include: Bump-up version to 1.4

This patch updates OpenSBI version to 1.4 as part of
release preparation.

Signed-off-by: Anup Patel <apatel@ventanamicro.com>
Signed-off-by: Anup Patel <anup@brainfault.org>

.gitignore

@@ -1,3 +1,10 @@
+# ignore anything begin with dot
+.*
+
+# exceptions we need even begin with dot
+!.clang-format
+!.gitignore
+
 # Object files
 *.o
 *.a
@@ -10,4 +17,3 @@ install/
 # Development friendly files
 tags
 cscope*
-*.swp

Makefile

@@ -168,7 +168,7 @@ endif
 OPENSBI_LD_PIE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) $(USE_LD_FLAG) -fPIE -nostdlib -Wl,-pie -x c /dev/null -o /dev/null >/dev/null 2>&1 && echo y || echo n)
 
 # Check whether the compiler supports -m(no-)save-restore
-CC_SUPPORT_SAVE_RESTORE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -mno-save-restore -x c /dev/null -o /dev/null 2>&1 | grep "\-save\-restore" >/dev/null && echo n || echo y)
+CC_SUPPORT_SAVE_RESTORE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -mno-save-restore -x c /dev/null -o /dev/null 2>&1 | grep -e "-save-restore" >/dev/null && echo n || echo y)
 
 # Check whether the assembler and the compiler support the Zicsr and Zifencei extensions
 CC_SUPPORT_ZICSR_ZIFENCEI := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -march=rv$(OPENSBI_CC_XLEN)imafd_zicsr_zifencei -x c /dev/null -o /dev/null 2>&1 | grep "zicsr\|zifencei" > /dev/null && echo n || echo y)
@@ -254,6 +254,7 @@ deps-y=$(platform-objs-path-y:.o=.dep)
 deps-y+=$(libsbi-objs-path-y:.o=.dep)
 deps-y+=$(libsbiutils-objs-path-y:.o=.dep)
 deps-y+=$(firmware-objs-path-y:.o=.dep)
+deps-y+=$(firmware-elfs-path-y:=.dep)
 
 # Setup platform ABI, ISA and Code Model
 ifndef PLATFORM_RISCV_ABI
@@ -330,7 +331,12 @@ GENFLAGS	+=	$(libsbiutils-genflags-y)
 GENFLAGS	+=	$(platform-genflags-y)
 GENFLAGS	+=	$(firmware-genflags-y)
 
-CFLAGS		=	-g -Wall -Werror -ffreestanding -nostdlib -fno-stack-protector -fno-strict-aliasing -O2
+CFLAGS		=	-g -Wall -Werror -ffreestanding -nostdlib -fno-stack-protector -fno-strict-aliasing
+ifneq ($(DEBUG),)
+CFLAGS		+=	-O0
+else
+CFLAGS		+=	-O2
+endif
 CFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls -mstrict-align
 # enable -m(no-)save-restore option by CC_SUPPORT_SAVE_RESTORE
 ifeq ($(CC_SUPPORT_SAVE_RESTORE),y)
@@ -369,7 +375,8 @@ ASFLAGS		+=	$(firmware-asflags-y)
 ARFLAGS		=	rcs
 
 ELFFLAGS	+=	$(USE_LD_FLAG)
-ELFFLAGS	+=	-Wl,--build-id=none -Wl,-N
+ELFFLAGS	+=	-Wl,--exclude-libs,ALL
+ELFFLAGS	+=	-Wl,--build-id=none
 ELFFLAGS	+=	$(platform-ldflags-y)
 ELFFLAGS	+=	$(firmware-ldflags-y)
 
@@ -395,10 +402,10 @@ merge_deps = $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     cat $(2) > $(1)
 copy_file =  $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     echo " COPY      $(subst $(build_dir)/,,$(1))"; \
-	     cp -f $(2) $(1)
+	     cp -L -f $(2) $(1)
 inst_file =  $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     echo " INSTALL   $(subst $(install_root_dir)/,,$(1))"; \
-	     cp -f $(2) $(1)
+	     cp -L -f $(2) $(1)
 inst_file_list = $(CMD_PREFIX)if [ ! -z "$(4)" ]; then \
 	     mkdir -p $(1)/$(3); \
 	     for file in $(4) ; do \
@@ -407,12 +414,17 @@ inst_file_list = $(CMD_PREFIX)if [ ! -z "$(4)" ]; then \
 	     dest_dir=`dirname $$dest_file`; \
 	     echo " INSTALL   "$(3)"/"`echo $$rel_file`; \
 	     mkdir -p $$dest_dir; \
-	     cp -f $$file $$dest_file; \
+	     cp -L -f $$file $$dest_file; \
 	     done \
 	     fi
 inst_header_dir =  $(CMD_PREFIX)mkdir -p $(1); \
 	     echo " INSTALL   $(subst $(install_root_dir)/,,$(1))"; \
-	     cp -rf $(2) $(1)
+	     cp -L -rf $(2) $(1)
+compile_cpp_dep = $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
+	     echo " CPP-DEP   $(subst $(build_dir)/,,$(1))"; \
+	     printf %s `dirname $(1)`/  > $(1) && \
+	     $(CC) $(CPPFLAGS) -x c -MM $(3) \
+	       -MT `basename $(1:.dep=$(2))` >> $(1) || rm -f $(1)
 compile_cpp = $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     echo " CPP       $(subst $(build_dir)/,,$(1))"; \
 	     $(CPP) $(CPPFLAGS) -x c $(2) | grep -v "\#" > $(1)
@@ -543,6 +555,9 @@ $(platform_build_dir)/%.bin: $(platform_build_dir)/%.elf
 $(platform_build_dir)/%.elf: $(platform_build_dir)/%.o $(platform_build_dir)/%.elf.ld $(platform_build_dir)/lib/libplatsbi.a
 	$(call compile_elf,$@,$@.ld,$< $(platform_build_dir)/lib/libplatsbi.a)
 
+$(platform_build_dir)/%.dep: $(src_dir)/%.ldS $(KCONFIG_CONFIG)
+	$(call compile_cpp_dep,$@,.ld,$<)
+
 $(platform_build_dir)/%.ld: $(src_dir)/%.ldS
 	$(call compile_cpp,$@,$<)
 

README.md

@@ -1,11 +1,15 @@
 RISC-V Open Source Supervisor Binary Interface (OpenSBI)
 ========================================================
 
+![RISC-V OpenSBI](docs/riscv_opensbi_logo_final_color.png)
+
 Copyright and License
 ---------------------
 
-The OpenSBI project is copyright (c) 2019 Western Digital Corporation
-or its affiliates and other contributors.
+The OpenSBI project is:
+
+* Copyright (c) 2019 Western Digital Corporation or its affiliates
+* Copyright (c) 2023 RISC-V International
 
 It is distributed under the terms of the BSD 2-clause license
 ("Simplified BSD License" or "FreeBSD License", SPDX: *BSD-2-Clause*).
@@ -298,6 +302,19 @@ NOTE: Using `BUILD_INFO=y` without specifying SOURCE_DATE_EPOCH will violate
 purpose, and should NOT be used in a product which follows "reproducible
 builds".
 
+Building with optimization off for debugging
+--------------------------------------------
+
+When debugging OpenSBI, we may want to turn off the compiler optimization and
+make debugging produce the expected results for a better debugging experience.
+To build with optimization off we can just simply add `DEBUG=1`, like:
+```
+make DEBUG=1
+```
+
+This definition is ONLY for development and debug purpose, and should NOT be
+used in a product build.
+
 Contributing to OpenSBI
 -----------------------
 

docs/domain_support.md

@@ -52,6 +52,7 @@ has following details:
 * **next_mode** - Privilege mode of the next booting stage for this
   domain. This can be either S-mode or U-mode.
 * **system_reset_allowed** - Is domain allowed to reset the system?
+* **system_suspend_allowed** - Is domain allowed to suspend the system?
 
 The memory regions represented by **regions** in **struct sbi_domain** have
 following additional constraints to align with RISC-V PMP requirements:
@@ -91,6 +92,7 @@ following manner:
 * **next_mode** - Next booting stage mode in coldboot HART scratch space
   is the next mode for the ROOT domain
 * **system_reset_allowed** - The ROOT domain is allowed to reset the system
+* **system_suspend_allowed** - The ROOT domain is allowed to suspend the system
 
 Domain Effects
 --------------
@@ -124,6 +126,9 @@ The DT properties of a domain configuration DT node are as follows:
 * **compatible** (Mandatory) - The compatible string of the domain
   configuration. This DT property should have value *"opensbi,domain,config"*
 
+* **system-suspend-test** (Optional) - When present, enable a system
+  suspend test implementation which simply waits five seconds and issues a WFI.
+
 ### Domain Memory Region Node
 
 The domain memory region DT node describes details of a memory region and
@@ -160,8 +165,16 @@ The DT properties of a domain instance DT node are as follows:
 * **regions** (Optional) - The list of domain memory region DT node phandle
   and access permissions for the domain instance. Each list entry is a pair
   of DT node phandle and access permissions. The access permissions are
-  represented as a 32bit bitmask having bits: **readable** (BIT[0]),
-  **writeable** (BIT[1]), **executable** (BIT[2]), and **m-mode** (BIT[3]).
+  represented as a 32bit bitmask having bits: **M readable** (BIT[0]),
+  **M writeable** (BIT[1]), **M executable** (BIT[2]), **SU readable**
+  (BIT[3]), **SU writable** (BIT[4]), and **SU executable** (BIT[5]).
+  The enforce permission bit (BIT[6]), if set, will lock the permissions
+  in the PMP. This will enforce the permissions on M-mode as well which
+  otherwise will have unrestricted access. This bit must be used with
+  caution because no changes can be made to a PMP entry once its locked
+  until the hart is reset.
+  Any region of a domain defined in DT node cannot have only M-bits set
+  in access permissions i.e. it cannot be an m-mode only accessible region.
 * **boot-hart** (Optional) - The DT node phandle of the HART booting the
   domain instance. If coldboot HART is assigned to the domain instance then
   this DT property is ignored and the coldboot HART is assumed to be the
@@ -180,13 +193,15 @@ The DT properties of a domain instance DT node are as follows:
   is used as default value.
 * **next-mode** (Optional) - The 32 bit next booting stage mode for the
   domain instance. The possible values of this DT property are: **0x1**
-  (s-mode), and **0x0** (u-mode). If this DT property is not available
+  (S-mode), and **0x0** (U-mode). If this DT property is not available
   and coldboot HART is not assigned to the domain instance then **0x1**
   is used as default value. If this DT property is not available and
   coldboot HART is assigned to the domain instance then **next booting
   stage mode of coldboot HART** is used as default value.
 * **system-reset-allowed** (Optional) - A boolean flag representing
   whether the domain instance is allowed to do system reset.
+* **system-suspend-allowed** (Optional) - A boolean flag representing
+  whether the domain instance is allowed to do system suspend.
 
 ### Assigning HART To Domain Instance
 
@@ -195,9 +210,9 @@ platform support can provide the HART to domain instance assignment using
 platform specific callback.
 
 The HART to domain instance assignment can be parsed from the device tree
-using optional DT property **opensbi,domain** in each CPU DT node. The
-value of DT property **opensbi,domain** is the DT phandle of the domain
-instance DT node. If **opensbi,domain** DT property is not specified then
+using optional DT property **opensbi-domain** in each CPU DT node. The
+value of DT property **opensbi-domain** is the DT phandle of the domain
+instance DT node. If **opensbi-domain** DT property is not specified then
 corresponding HART is assigned to **the ROOT domain**.
 
 ### Domain Configuration Only Accessible to OpenSBI
@@ -222,6 +237,7 @@ be done:
     chosen {
         opensbi-domains {
             compatible = "opensbi,domain,config";
+            system-suspend-test;
 
             tmem: tmem {
                 compatible = "opensbi,domain,memregion";
@@ -246,18 +262,19 @@ be done:
             tdomain: trusted-domain {
                 compatible = "opensbi,domain,instance";
                 possible-harts = <&cpu0>;
-                regions = <&tmem 0x7>, <&tuart 0x7>;
+                regions = <&tmem 0x3f>, <&tuart 0x3f>;
                 boot-hart = <&cpu0>;
                 next-arg1 = <0x0 0x0>;
                 next-addr = <0x0 0x80100000>;
                 next-mode = <0x0>;
                 system-reset-allowed;
+                system-suspend-allowed;
             };
 
             udomain: untrusted-domain {
                 compatible = "opensbi,domain,instance";
                 possible-harts = <&cpu1 &cpu2 &cpu3 &cpu4>;
-                regions = <&tmem 0x0>, <&tuart 0x0>, <&allmem 0x7>;
+                regions = <&tmem 0x0>, <&tuart 0x0>, <&allmem 0x3f>;
             };
         };
     };

docs/firmware/fw.md

@@ -61,7 +61,7 @@ Firmware Configuration and Compilation
 All firmware types support the following common compile time configuration
 parameters:
 
-* **FW_TEXT_ADDR** - Defines the execution address of the OpenSBI firmware.
+* **FW_TEXT_START** - Defines the execution address of the OpenSBI firmware.
   This configuration parameter is mandatory.
 * **FW_FDT_PATH** - Path to an external flattened device tree binary file to
   be embedded in the *.rodata* section of the final firmware. If this option

docs/firmware/fw_jump.md

@@ -43,18 +43,18 @@ follows:
 
   When using the default *FW_JUMP_FDT_ADDR* with *PLATFORM=generic*, you must
   ensure *FW_JUMP_FDT_ADDR* is set high enough to avoid overwriting the kernel.
-  You can use the following method.
+  You can use the following method (e.g., using bash or zsh):
 
   ```
-  ${CROSS_COMPILE}objdump -h $KERNEL_ELF | sort -k 5,5 | awk -n '/^ +[0-9]+ /\
-  {addr="0x"$3; size="0x"$5; printf "0x""%x\n",addr+size}' \
-  | (( `tail -1` > 0x2200000 )) && echo fdt overlaps kernel,\
-  increase FW_JUMP_FDT_ADDR
+  ${CROSS_COMPILE}objdump -h $KERNEL_ELF | sort -k 5,5 | awk -n '
+  /^ +[0-9]+ / {addr="0x"$3; size="0x"$5; printf "0x""%x\n",addr+size}' |
+  (( `tail -1` > (FW_JUMP_FDT_ADDR - FW_JUMP_ADDR) )) &&
+  echo fdt overlaps kernel, increase FW_JUMP_FDT_ADDR
 
-  ${LLVM}objdump -h --show-lma $KERNEL_ELF | sort -k 5,5 | \
-  awk -n '/^ +[0-9]+ / {addr="0x"$3; size="0x"$5; printf "0x""%x\n",addr+size}'\
-  | (( `tail -1` > 0x2200000 )) && echo fdt overlaps kernel,\
-  increase FW_JUMP_FDT_ADDR
+  ${LLVM}objdump -h --show-lma $KERNEL_ELF | sort -k 5,5 | awk -n '
+  /^ +[0-9]+ / {addr="0x"$3; size="0x"$5; printf "0x""%x\n",addr+size}' |
+  (( `tail -1` > (FW_JUMP_FDT_ADDR - FW_JUMP_ADDR) )) &&
+  echo fdt overlaps kernel, increase FW_JUMP_FDT_ADDR
   ```
 
 *FW_JUMP* Example

docs/firmware/fw_payload.md

@@ -36,7 +36,7 @@ options. These configuration parameters can be defined using either the top
 level `make` command line or the target platform *objects.mk* configuration
 file. The parameters currently defined are as follows:
 
-* **FW_PAYLOAD_OFFSET** - Offset from *FW_TEXT_BASE* where the payload binary
+* **FW_PAYLOAD_OFFSET** - Offset from *FW_TEXT_START* where the payload binary
   will be linked in the final *FW_PAYLOAD* firmware binary image. This
   configuration parameter is mandatory if *FW_PAYLOAD_ALIGN* is not defined.
   Compilation errors will result from an incorrect definition of

docs/platform/generic.md

@@ -53,7 +53,7 @@ RISC-V Platforms Using Generic Platform
 * **Spike** (*[spike.md]*)
 * **T-HEAD C9xx series Processors** (*[thead-c9xx.md]*)
 
-[andes-ae350.md]: andse-ae350.md
+[andes-ae350.md]: andes-ae350.md
 [qemu_virt.md]: qemu_virt.md
 [renesas-rzfive.md]: renesas-rzfive.md
 [shakti_cclass.md]: shakti_cclass.md

docs/platform/thead-c9xx.md

@@ -1,7 +1,7 @@
 T-HEAD C9xx Series Processors
 =============================
 
-The **C9xx** series processors are high-performance RISC-V architecture
+The C9xx series processors are high-performance RISC-V architecture
 multi-core processors with AI vector acceleration engine.
 
 For more details, refer [T-HEAD.CN](https://www.t-head.cn/)
@@ -12,185 +12,16 @@ To build the platform-specific library and firmware images, provide the
 Platform Options
 ----------------
 
-The *T-HEAD C9xx* does not have any platform-specific compile options
+The T-HEAD C9xx does not have any platform-specific compile options
 because it uses generic platform.
 
 ```
-CROSS_COMPILE=riscv64-linux-gnu- PLATFORM=generic /usr/bin/make
+CROSS_COMPILE=riscv64-linux-gnu- PLATFORM=generic make
 ```
 
-The *T-HEAD C9xx* DTB provided to OpenSBI generic firmwares will usually have
-"riscv,clint0", "riscv,plic0", "thead,reset-sample" compatible strings.
+Here is the simplest boot flow for a fpga prototype:
 
-DTS Example1: (Single core, eg: Allwinner D1 - c906)
-----------------------------------------------------
+ (Jtag gdbinit) -> (zsb) -> (opensbi) -> (linux)
 
-```
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		timebase-frequency = <3000000>;
-		cpu@0 {
-			device_type = "cpu";
-			reg = <0>;
-			status = "okay";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdcv";
-			mmu-type = "riscv,sv39";
-			cpu0_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-	};
-
-	soc {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		compatible = "simple-bus";
-		ranges;
-
-		clint0: clint@14000000 {
-			compatible = "allwinner,sun20i-d1-clint";
-			interrupts-extended = <
-				&cpu0_intc  3 &cpu0_intc  7
-				>;
-			reg = <0x0 0x14000000 0x0 0x04000000>;
-		};
-
-		intc: interrupt-controller@10000000 {
-			#interrupt-cells = <1>;
-			compatible = "allwinner,sun20i-d1-plic",
-				     "thead,c900-plic";
-			interrupt-controller;
-			interrupts-extended = <
-				&cpu0_intc  0xffffffff &cpu0_intc  9
-				>;
-			reg = <0x0 0x10000000 0x0 0x04000000>;
-			reg-names = "control";
-			riscv,max-priority = <7>;
-			riscv,ndev = <200>;
-		};
-	}
-```
-
-DTS Example2: (Multi cores with soc reset-regs)
------------------------------------------------
-
-```
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		timebase-frequency = <3000000>;
-		cpu@0 {
-			device_type = "cpu";
-			reg = <0>;
-			status = "okay";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu0_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-		cpu@1 {
-			device_type = "cpu";
-			reg = <1>;
-			status = "fail";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu1_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-		cpu@2 {
-			device_type = "cpu";
-			reg = <2>;
-			status = "fail";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu2_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-		cpu@3 {
-			device_type = "cpu";
-			reg = <3>;
-			status = "fail";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu3_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-	};
-
-	soc {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		compatible = "simple-bus";
-		ranges;
-
-		reset: reset-sample {
-			compatible = "thead,reset-sample";
-			entry-reg = <0xff 0xff019050>;
-			entry-cnt = <4>;
-			control-reg = <0xff 0xff015004>;
-			control-val = <0x1c>;
-			csr-copy = <0x7f3 0x7c0 0x7c1 0x7c2 0x7c3 0x7c5 0x7cc>;
-		};
-
-		clint0: clint@ffdc000000 {
-			compatible = "riscv,clint0";
-			interrupts-extended = <
-				&cpu0_intc  3 &cpu0_intc  7
-				&cpu1_intc  3 &cpu1_intc  7
-				&cpu2_intc  3 &cpu2_intc  7
-				&cpu3_intc  3 &cpu3_intc  7
-				&cpu4_intc  3 &cpu4_intc  7
-				>;
-			reg = <0xff 0xdc000000 0x0 0x04000000>;
-		};
-
-		intc: interrupt-controller@ffd8000000 {
-			#interrupt-cells = <1>;
-			compatible = "thead,c900-plic";
-			interrupt-controller;
-			interrupts-extended = <
-				&cpu0_intc  0xffffffff &cpu0_intc  9
-				&cpu1_intc  0xffffffff &cpu1_intc  9
-				&cpu2_intc  0xffffffff &cpu2_intc  9
-				&cpu3_intc  0xffffffff &cpu3_intc  9
-				>;
-			reg = <0xff 0xd8000000 0x0 0x04000000>;
-			reg-names = "control";
-			riscv,max-priority = <7>;
-			riscv,ndev = <80>;
-		};
-	}
-```
-
-DTS Example2: (Multi cores with old reset csrs)
------------------------------------------------
-```
-	reset: reset-sample {
-		compatible = "thead,reset-sample";
-		using-csr-reset;
-		csr-copy = <0x7c0 0x7c1 0x7c2 0x7c3 0x7c5 0x7cc
-			    0x3b0 0x3b1 0x3b2 0x3b3
-			    0x3b4 0x3b5 0x3b6 0x3b7
-			    0x3a0>;
-	};
-```
+For more details, refer:
+ [zero stage boot](https://github.com/c-sky/zero_stage_boot)

docs/platform_requirements.md

@@ -18,7 +18,7 @@ Base Platform Requirements
 
 The base RISC-V platform requirements for OpenSBI are as follows:
 
-1. At least rv32ima or rv64ima required on all HARTs
+1. At least rv32ima_zicsr or rv64ima_zicsr required on all HARTs
 2. At least one HART should have S-mode support because:
 
      * SBI calls are meant for RISC-V S-mode (Supervisor mode)
@@ -33,7 +33,7 @@ The base RISC-V platform requirements for OpenSBI are as follows:
 6. Hardware support for injecting M-mode software interrupts on
    a multi-HART platform
 
-The RISC-V extensions not covered by rv32ima or rv64ima are optional
+The RISC-V extensions not covered by rv32ima_zicsr or rv64ima_zicsr are optional
 for OpenSBI. Although, OpenSBI will detect and handle some of these
 optional RISC-V extensions at runtime.
 

docs/pmu_support.md

@@ -125,3 +125,85 @@ pmu {
 					  <0x0 0x2 0xffffffff 0xffffe0ff 0x18>;
 };
 ```
+
+### Example 3
+
+```
+/*
+ * For Andes 45-series platforms. The encodings can be found in the
+ * "Machine Performance Monitoring Event Selector" section
+ * http://www.andestech.com/wp-content/uploads/AX45MP-1C-Rev.-5.0.0-Datasheet.pdf
+ */
+pmu {
+	compatible 			= "riscv,pmu";
+	riscv,event-to-mhpmevent =
+					 <0x1 0x0000 0x10>, /* CPU_CYCLES          -> Cycle count */
+					 <0x2 0x0000 0x20>, /* INSTRUCTIONS        -> Retired instruction count */
+					 <0x3 0x0000 0x41>, /* CACHE_REFERENCES    -> D-Cache access */
+					 <0x4 0x0000 0x51>, /* CACHE_MISSES        -> D-Cache miss */
+					 <0x5 0x0000 0x80>, /* BRANCH_INSTRUCTIONS -> Conditional branch instruction count */
+					 <0x6 0x0000 0x02>, /* BRANCH_MISSES       -> Misprediction of conditional branches */
+					 <0x10000 0x0000 0x61>,  /* L1D_READ_ACCESS  -> D-Cache load access */
+					 <0x10001 0x0000 0x71>,  /* L1D_READ_MISS    -> D-Cache load miss */
+					 <0x10002 0x0000 0x81>,  /* L1D_WRITE_ACCESS -> D-Cache store access */
+					 <0x10003 0x0000 0x91>,  /* L1D_WRITE_MISS   -> D-Cache store miss */
+					 <0x10008 0x0000 0x21>,  /* L1I_READ_ACCESS  -> I-Cache access */
+					 <0x10009 0x0000 0x31>;  /* L1I_READ_MISS    -> I-Cache miss */
+	riscv,event-to-mhpmcounters = <0x1 0x6 0x78>,
+							<0x10000 0x10003 0x78>,
+							<0x10008 0x10009 0x78>;
+	riscv,raw-event-to-mhpmcounters =
+						<0x0 0x10 0xffffffff 0xffffffff 0x78>, /* Cycle count */
+						<0x0 0x20 0xffffffff 0xffffffff 0x78>, /* Retired instruction count */
+						<0x0 0x30 0xffffffff 0xffffffff 0x78>, /* Integer load instruction count */
+						<0x0 0x40 0xffffffff 0xffffffff 0x78>, /* Integer store instruction count */
+						<0x0 0x50 0xffffffff 0xffffffff 0x78>, /* Atomic instruction count */
+						<0x0 0x60 0xffffffff 0xffffffff 0x78>, /* System instruction count */
+						<0x0 0x70 0xffffffff 0xffffffff 0x78>, /* Integer computational instruction count */
+						<0x0 0x80 0xffffffff 0xffffffff 0x78>, /* Conditional branch instruction count */
+						<0x0 0x90 0xffffffff 0xffffffff 0x78>, /* Taken conditional branch instruction count */
+						<0x0 0xA0 0xffffffff 0xffffffff 0x78>, /* JAL instruction count */
+						<0x0 0xB0 0xffffffff 0xffffffff 0x78>, /* JALR instruction count */
+						<0x0 0xC0 0xffffffff 0xffffffff 0x78>, /* Return instruction count */
+						<0x0 0xD0 0xffffffff 0xffffffff 0x78>, /* Control transfer instruction count */
+						<0x0 0xE0 0xffffffff 0xffffffff 0x78>, /* EXEC.IT instruction count */
+						<0x0 0xF0 0xffffffff 0xffffffff 0x78>, /* Integer multiplication instruction count */
+						<0x0 0x100 0xffffffff 0xffffffff 0x78>, /* Integer division instruction count */
+						<0x0 0x110 0xffffffff 0xffffffff 0x78>, /* Floating-point load instruction count */
+						<0x0 0x120 0xffffffff 0xffffffff 0x78>, /* Floating-point store instruction count */
+						<0x0 0x130 0xffffffff 0xffffffff 0x78>, /* Floating-point addition/subtraction instruction count */
+						<0x0 0x140 0xffffffff 0xffffffff 0x78>, /* Floating-point multiplication instruction count */
+						<0x0 0x150 0xffffffff 0xffffffff 0x78>, /* Floating-point fused multiply-add instruction count */
+						<0x0 0x160 0xffffffff 0xffffffff 0x78>, /* Floating-point division or square-root instruction count */
+						<0x0 0x170 0xffffffff 0xffffffff 0x78>, /* Other floating-point instruction count */
+						<0x0 0x180 0xffffffff 0xffffffff 0x78>, /* Integer multiplication and add/sub instruction count */
+						<0x0 0x190 0xffffffff 0xffffffff 0x78>, /* Retired operation count */
+						<0x0 0x01 0xffffffff 0xffffffff 0x78>, /* ILM access */
+						<0x0 0x11 0xffffffff 0xffffffff 0x78>, /* DLM access */
+						<0x0 0x21 0xffffffff 0xffffffff 0x78>, /* I-Cache access */
+						<0x0 0x31 0xffffffff 0xffffffff 0x78>, /* I-Cache miss */
+						<0x0 0x41 0xffffffff 0xffffffff 0x78>, /* D-Cache access */
+						<0x0 0x51 0xffffffff 0xffffffff 0x78>, /* D-Cache miss */
+						<0x0 0x61 0xffffffff 0xffffffff 0x78>, /* D-Cache load access */
+						<0x0 0x71 0xffffffff 0xffffffff 0x78>, /* D-Cache load miss */
+						<0x0 0x81 0xffffffff 0xffffffff 0x78>, /* D-Cache store access */
+						<0x0 0x91 0xffffffff 0xffffffff 0x78>, /* D-Cache store miss */
+						<0x0 0xA1 0xffffffff 0xffffffff 0x78>, /* D-Cache writeback */
+						<0x0 0xB1 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for I-Cache fill data */
+						<0x0 0xC1 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for D-Cache fill data */
+						<0x0 0xD1 0xffffffff 0xffffffff 0x78>, /* Uncached fetch data access from bus */
+						<0x0 0xE1 0xffffffff 0xffffffff 0x78>, /* Uncached load data access from bus */
+						<0x0 0xF1 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for uncached fetch data from bus */
+						<0x0 0x101 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for uncached load data from bus */
+						<0x0 0x111 0xffffffff 0xffffffff 0x78>, /* Main ITLB access */
+						<0x0 0x121 0xffffffff 0xffffffff 0x78>, /* Main ITLB miss */
+						<0x0 0x131 0xffffffff 0xffffffff 0x78>, /* Main DTLB access */
+						<0x0 0x141 0xffffffff 0xffffffff 0x78>, /* Main DTLB miss */
+						<0x0 0x151 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for Main ITLB fill data */
+						<0x0 0x161 0xffffffff 0xffffffff 0x78>, /* Pipeline stall cycles caused by Main DTLB miss */
+						<0x0 0x171 0xffffffff 0xffffffff 0x78>, /* Hardware prefetch bus access */
+						<0x0 0x02 0xffffffff 0xffffffff 0x78>, /* Misprediction of conditional branches */
+						<0x0 0x12 0xffffffff 0xffffffff 0x78>, /* Misprediction of taken conditional branches */
+						<0x0 0x22 0xffffffff 0xffffffff 0x78>; /* Misprediction of targets of Return instructions */
+};
+```

firmware/fw_base.S

@@ -79,43 +79,19 @@ _try_lottery:
 	lla	t0, __rel_dyn_start
 	lla	t1, __rel_dyn_end
 	beq	t0, t1, _relocate_done
-	j	5f
 2:
-	REG_L	t5, -(REGBYTES*2)(t0)	/* t5 <-- relocation info:type */
+	REG_L	t5, REGBYTES(t0)	/* t5 <-- relocation info:type */
 	li	t3, R_RISCV_RELATIVE	/* reloc type R_RISCV_RELATIVE */
 	bne	t5, t3, 3f
-	REG_L	t3, -(REGBYTES*3)(t0)
-	REG_L	t5, -(REGBYTES)(t0)	/* t5 <-- addend */
+	REG_L	t3, 0(t0)
+	REG_L	t5, (REGBYTES * 2)(t0)	/* t5 <-- addend */
 	add	t5, t5, t2
 	add	t3, t3, t2
 	REG_S	t5, 0(t3)		/* store runtime address to the GOT entry */
-	j	5f
 
 3:
-	lla	t4, __dyn_sym_start
-
-4:
-	REG_L	t5, -(REGBYTES*2)(t0)	/* t5 <-- relocation info:type */
-	srli	t6, t5, SYM_INDEX	/* t6 <--- sym table index */
-	andi	t5, t5, 0xFF		/* t5 <--- relocation type */
-	li	t3, RELOC_TYPE
-	bne	t5, t3, 5f
-
-	/* address R_RISCV_64 or R_RISCV_32 cases*/
-	REG_L	t3, -(REGBYTES*3)(t0)
-	li	t5, SYM_SIZE
-	mul	t6, t6, t5
-	add	s5, t4, t6
-	REG_L	t6, -(REGBYTES)(t0)	/* t0 <-- addend */
-	REG_L	t5, REGBYTES(s5)
-	add	t5, t5, t6
-	add	t5, t5, t2		/* t5 <-- location to fix up in RAM */
-	add	t3, t3, t2		/* t3 <-- location to fix up in RAM */
-	REG_S	t5, 0(t3)		/* store runtime address to the variable */
-
-5:
-	addi	t0, t0, (REGBYTES*3)
-	ble	t0, t1, 2b
+	addi	t0, t0, (REGBYTES * 3)
+	blt	t0, t1, 2b
 	j	_relocate_done
 _wait_relocate_copy_done:
 	j	_wait_for_boot_hart
@@ -257,20 +233,28 @@ _bss_zero:
 	/* Preload HART details
 	 * s7 -> HART Count
 	 * s8 -> HART Stack Size
+	 * s9 -> Heap Size
+	 * s10 -> Heap Offset
 	 */
 	lla	a4, platform
 #if __riscv_xlen > 32
 	lwu	s7, SBI_PLATFORM_HART_COUNT_OFFSET(a4)
 	lwu	s8, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(a4)
+	lwu	s9, SBI_PLATFORM_HEAP_SIZE_OFFSET(a4)
 #else
 	lw	s7, SBI_PLATFORM_HART_COUNT_OFFSET(a4)
 	lw	s8, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(a4)
+	lw	s9, SBI_PLATFORM_HEAP_SIZE_OFFSET(a4)
 #endif
 
 	/* Setup scratch space for all the HARTs*/
 	lla	tp, _fw_end
 	mul	a5, s7, s8
 	add	tp, tp, a5
+	/* Setup heap base address */
+	lla	s10, _fw_start
+	sub	s10, tp, s10
+	add	tp, tp, s9
 	/* Keep a copy of tp */
 	add	t3, tp, zero
 	/* Counter */
@@ -285,8 +269,11 @@ _scratch_init:
 	 * t3 -> the firmware end address
 	 * s7 -> HART count
 	 * s8 -> HART stack size
+	 * s9 -> Heap Size
+	 * s10 -> Heap Offset
 	 */
 	add	tp, t3, zero
+	sub	tp, tp, s9
 	mul	a5, s8, t1
 	sub	tp, tp, a5
 	li	a5, SBI_SCRATCH_SIZE
@@ -298,6 +285,16 @@ _scratch_init:
 	sub	a5, t3, a4
 	REG_S	a4, SBI_SCRATCH_FW_START_OFFSET(tp)
 	REG_S	a5, SBI_SCRATCH_FW_SIZE_OFFSET(tp)
+
+	/* Store R/W section's offset in scratch space */
+	lla	a5, _fw_rw_start
+	sub	a5, a5, a4
+	REG_S	a5, SBI_SCRATCH_FW_RW_OFFSET(tp)
+
+	/* Store fw_heap_offset and fw_heap_size in scratch space */
+	REG_S	s10, SBI_SCRATCH_FW_HEAP_OFFSET(tp)
+	REG_S	s9, SBI_SCRATCH_FW_HEAP_SIZE_OFFSET(tp)
+
 	/* Store next arg1 in scratch space */
 	MOV_3R	s0, a0, s1, a1, s2, a2
 	call	fw_next_arg1
@@ -402,8 +399,8 @@ _fdt_reloc_done:
 	/* mark boot hart done */
 	li	t0, BOOT_STATUS_BOOT_HART_DONE
 	lla	t1, _boot_status
-	REG_S	t0, 0(t1)
 	fence	rw, rw
+	REG_S	t0, 0(t1)
 	j	_start_warm
 
 	/* waiting for boot hart to be done (_boot_status == 2) */
@@ -412,9 +409,9 @@ _wait_for_boot_hart:
 	lla	t1, _boot_status
 	REG_L	t1, 0(t1)
 	/* Reduce the bus traffic so that boot hart may proceed faster */
-	nop
-	nop
-	nop
+	div	t2, t2, zero
+	div	t2, t2, zero
+	div	t2, t2, zero
 	bne	t0, t1, _wait_for_boot_hart
 
 _start_warm:
@@ -422,9 +419,8 @@ _start_warm:
 	li	ra, 0
 	call	_reset_regs
 
-	/* Disable and clear all interrupts */
+	/* Disable all interrupts */
 	csrw	CSR_MIE, zero
-	csrw	CSR_MIP, zero
 
 	/* Find HART count and HART stack size */
 	lla	a4, platform
@@ -453,7 +449,6 @@ _start_warm:
 	add	s9, s9, 4
 	add	a4, a4, 1
 	blt	a4, s7, 1b
-	li	a4, -1
 2:	add	s6, a4, zero
 3:	bge	s6, s7, _start_hang
 

firmware/fw_base.ldS

@@ -30,17 +30,34 @@
 
 	/* Beginning of the read-only data sections */
 
+	PROVIDE(_rodata_start = .);
+
 	.rodata :
 	{
-		PROVIDE(_rodata_start = .);
 		*(.rodata .rodata.*)
 		. = ALIGN(8);
-		PROVIDE(_rodata_end = .);
 	}
 
+	. = ALIGN(0x1000); /* Ensure next section is page aligned */
+
+	.rela.dyn : {
+		PROVIDE(__rel_dyn_start = .);
+		*(.rela*)
+		PROVIDE(__rel_dyn_end = .);
+	}
+
+	PROVIDE(_rodata_end = .);
+
 	/* End of the read-only data sections */
 
-	. = ALIGN(0x1000); /* Ensure next section is page aligned */
+	/*
+	 * PMP regions must be to be power-of-2. RX/RW will have separate
+	 * regions, so ensure that the split is power-of-2.
+	 */
+	. = ALIGN(1 << LOG2CEIL((SIZEOF(.rodata) + SIZEOF(.text)
+				+ SIZEOF(.dynsym) + SIZEOF(.rela.dyn))));
+
+	PROVIDE(_fw_rw_start = .);
 
 	/* Beginning of the read-write data sections */
 
@@ -59,19 +76,6 @@
 		PROVIDE(_data_end = .);
 	}
 
-	.dynsym : {
-		PROVIDE(__dyn_sym_start = .);
-		*(.dynsym)
-		PROVIDE(__dyn_sym_end = .);
-	}
-
-	.rela.dyn : {
-		PROVIDE(__rel_dyn_start = .);
-		*(.rela*)
-		. = ALIGN(8);
-		PROVIDE(__rel_dyn_end = .);
-	}
-
 	. = ALIGN(0x1000); /* Ensure next section is page aligned */
 
 	.bss :

firmware/fw_dynamic.S

@@ -129,7 +129,7 @@ fw_options:
 	REG_L	a0, (a0)
 	ret
 
-	.section .entry, "ax", %progbits
+	.section .data
 	.align 3
 _dynamic_next_arg1:
 	RISCV_PTR 0x0

firmware/fw_jump.S

@@ -90,7 +90,7 @@ fw_options:
 #error "Must define FW_JUMP_ADDR"
 #endif
 
-	.section .entry, "ax", %progbits
+	.section .rodata
 	.align 3
 _jump_addr:
 	RISCV_PTR FW_JUMP_ADDR

firmware/payloads/test_head.S

@@ -78,7 +78,7 @@ _start_hang:
 	wfi
 	j	_start_hang
 
-	.section .entry, "ax", %progbits
+	.section .data
 	.align	3
 _hart_lottery:
 	RISCV_PTR	0

firmware/payloads/test_main.c

@@ -8,31 +8,42 @@
  */
 
 #include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_string.h>
 
-#define SBI_ECALL(__eid, __fid, __a0, __a1, __a2)                             \
-	({                                                                    \
-		register unsigned long a0 asm("a0") = (unsigned long)(__a0);  \
-		register unsigned long a1 asm("a1") = (unsigned long)(__a1);  \
-		register unsigned long a2 asm("a2") = (unsigned long)(__a2);  \
-		register unsigned long a6 asm("a6") = (unsigned long)(__fid); \
-		register unsigned long a7 asm("a7") = (unsigned long)(__eid); \
-		asm volatile("ecall"                                          \
-			     : "+r"(a0)                                       \
-			     : "r"(a1), "r"(a2), "r"(a6), "r"(a7)             \
-			     : "memory");                                     \
-		a0;                                                           \
-	})
+struct sbiret {
+	unsigned long error;
+	unsigned long value;
+};
 
-#define SBI_ECALL_0(__eid, __fid) SBI_ECALL(__eid, __fid, 0, 0, 0)
-#define SBI_ECALL_1(__eid, __fid, __a0) SBI_ECALL(__eid, __fid, __a0, 0, 0)
-#define SBI_ECALL_2(__eid, __fid, __a0, __a1) SBI_ECALL(__eid, __fid, __a0, __a1, 0)
+struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
+			unsigned long arg1, unsigned long arg2,
+			unsigned long arg3, unsigned long arg4,
+			unsigned long arg5)
+{
+	struct sbiret ret;
 
-#define sbi_ecall_console_putc(c) SBI_ECALL_1(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, (c))
+	register unsigned long a0 asm ("a0") = (unsigned long)(arg0);
+	register unsigned long a1 asm ("a1") = (unsigned long)(arg1);
+	register unsigned long a2 asm ("a2") = (unsigned long)(arg2);
+	register unsigned long a3 asm ("a3") = (unsigned long)(arg3);
+	register unsigned long a4 asm ("a4") = (unsigned long)(arg4);
+	register unsigned long a5 asm ("a5") = (unsigned long)(arg5);
+	register unsigned long a6 asm ("a6") = (unsigned long)(fid);
+	register unsigned long a7 asm ("a7") = (unsigned long)(ext);
+	asm volatile ("ecall"
+		      : "+r" (a0), "+r" (a1)
+		      : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7)
+		      : "memory");
+	ret.error = a0;
+	ret.value = a1;
+
+	return ret;
+}
 
 static inline void sbi_ecall_console_puts(const char *str)
 {
-	while (str && *str)
-		sbi_ecall_console_putc(*str++);
+	sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_WRITE,
+		  sbi_strlen(str), (unsigned long)str, 0, 0, 0, 0);
 }
 
 #define wfi()                                             \

include/sbi/riscv_asm.h

@@ -181,6 +181,12 @@ int misa_xlen(void);
 /* Get RISC-V ISA string representation */
 void misa_string(int xlen, char *out, unsigned int out_sz);
 
+/* Disable pmp entry at a given index */
+int pmp_disable(unsigned int n);
+
+/* Check if the matching field is set */
+int is_pmp_entry_mapped(unsigned long entry);
+
 int pmp_set(unsigned int n, unsigned long prot, unsigned long addr,
 	    unsigned long log2len);
 

include/sbi/riscv_atomic.h

@@ -39,14 +39,14 @@ unsigned int atomic_raw_xchg_uint(volatile unsigned int *ptr,
 unsigned long atomic_raw_xchg_ulong(volatile unsigned long *ptr,
 				    unsigned long newval);
 /**
- * Set a bit in an atomic variable and return the new value.
+ * Set a bit in an atomic variable and return the value of bit before modify.
  * @nr : Bit to set.
  * @atom: atomic variable to modify
  */
 int atomic_set_bit(int nr, atomic_t *atom);
 
 /**
- * Clear a bit in an atomic variable and return the new value.
+ * Clear a bit in an atomic variable and return the value of bit before modify.
  * @nr : Bit to set.
  * @atom: atomic variable to modify
  */
@@ -54,14 +54,14 @@ int atomic_set_bit(int nr, atomic_t *atom);
 int atomic_clear_bit(int nr, atomic_t *atom);
 
 /**
- * Set a bit in any address and return the new value .
+ * Set a bit in any address and return the value of bit before modify.
  * @nr : Bit to set.
  * @addr: Address to modify
  */
 int atomic_raw_set_bit(int nr, volatile unsigned long *addr);
 
 /**
- * Clear a bit in any address and return the new value .
+ * Clear a bit in any address and return the value of bit before modify.
  * @nr : Bit to set.
  * @addr: Address to modify
  */

include/sbi/riscv_elf.h

@@ -1,14 +1,6 @@
 #ifndef __RISCV_ELF_H__
 #define __RISCV_ELF_H__
 
-#include <sbi/riscv_asm.h>
-
-#define R_RISCV_32		1
-#define R_RISCV_64		2
 #define R_RISCV_RELATIVE	3
 
-#define RELOC_TYPE		__REG_SEL(R_RISCV_64, R_RISCV_32)
-#define SYM_INDEX		__REG_SEL(0x20,	0x8)
-#define SYM_SIZE		__REG_SEL(0x18,0x10)
-
 #endif

include/sbi/riscv_encoding.h

@@ -207,13 +207,8 @@
 
 #define MHPMEVENT_SSCOF_MASK		_ULL(0xFFFF000000000000)
 
-#if __riscv_xlen > 32
 #define ENVCFG_STCE			(_ULL(1) << 63)
 #define ENVCFG_PBMTE			(_ULL(1) << 62)
-#else
-#define ENVCFGH_STCE			(_UL(1) << 31)
-#define ENVCFGH_PBMTE			(_UL(1) << 30)
-#endif
 #define ENVCFG_CBZE			(_UL(1) << 7)
 #define ENVCFG_CBCFE			(_UL(1) << 6)
 #define ENVCFG_CBIE_SHIFT		4
@@ -430,6 +425,7 @@
 #define CSR_MARCHID			0xf12
 #define CSR_MIMPID			0xf13
 #define CSR_MHARTID			0xf14
+#define CSR_MCONFIGPTR			0xf15
 
 /* Machine Trap Setup */
 #define CSR_MSTATUS			0x300
@@ -602,6 +598,8 @@
 
 /* Machine Counter Setup */
 #define CSR_MCOUNTINHIBIT		0x320
+#define CSR_MCYCLECFG			0x321
+#define CSR_MINSTRETCFG			0x322
 #define CSR_MHPMEVENT3			0x323
 #define CSR_MHPMEVENT4			0x324
 #define CSR_MHPMEVENT5			0x325
@@ -633,6 +631,8 @@
 #define CSR_MHPMEVENT31			0x33f
 
 /* For RV32 */
+#define CSR_MCYCLECFGH			0x721
+#define CSR_MINSTRETCFGH		0x722
 #define CSR_MHPMEVENT3H			0x723
 #define CSR_MHPMEVENT4H			0x724
 #define CSR_MHPMEVENT5H			0x725
@@ -663,6 +663,21 @@
 #define CSR_MHPMEVENT30H		0x73e
 #define CSR_MHPMEVENT31H		0x73f
 
+/* Machine Security Configuration CSR (mseccfg) */
+#define CSR_MSECCFG			0x747
+#define CSR_MSECCFGH			0x757
+
+#define MSECCFG_MML_SHIFT		(0)
+#define MSECCFG_MML			(_UL(1) << MSECCFG_MML_SHIFT)
+#define MSECCFG_MMWP_SHIFT		(1)
+#define MSECCFG_MMWP			(_UL(1) << MSECCFG_MMWP_SHIFT)
+#define MSECCFG_RLB_SHIFT		(2)
+#define MSECCFG_RLB			(_UL(1) << MSECCFG_RLB_SHIFT)
+#define MSECCFG_USEED_SHIFT		(8)
+#define MSECCFG_USEED			(_UL(1) << MSECCFG_USEED_SHIFT)
+#define MSECCFG_SSEED_SHIFT		(9)
+#define MSECCFG_SSEED			(_UL(1) << MSECCFG_SSEED_SHIFT)
+
 /* Counter Overflow CSR */
 #define CSR_SCOUNTOVF			0xda0
 
@@ -736,6 +751,8 @@
 #define SMSTATEEN0_CS			(_ULL(1) << SMSTATEEN0_CS_SHIFT)
 #define SMSTATEEN0_FCSR_SHIFT		1
 #define SMSTATEEN0_FCSR			(_ULL(1) << SMSTATEEN0_FCSR_SHIFT)
+#define SMSTATEEN0_CONTEXT_SHIFT	57
+#define SMSTATEEN0_CONTEXT		(_ULL(1) << SMSTATEEN0_CONTEXT_SHIFT)
 #define SMSTATEEN0_IMSIC_SHIFT		58
 #define SMSTATEEN0_IMSIC		(_ULL(1) << SMSTATEEN0_IMSIC_SHIFT)
 #define SMSTATEEN0_AIA_SHIFT		59

include/sbi/riscv_fp.h

@@ -84,7 +84,7 @@
 #define GET_FFLAGS() csr_read(CSR_FFLAGS)
 #define SET_FFLAGS(value) csr_write(CSR_FFLAGS, (value))
 
-#define SET_FS_DIRTY() ((void)0)
+#define SET_FS_DIRTY(regs) (regs->mstatus |= MSTATUS_FS)
 
 #define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs))
 #define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs))
@@ -93,9 +93,9 @@
 #define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs))
 #define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs))
 #define SET_F32_RD(insn, regs, val) \
-	(SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY())
+	(SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY(regs))
 #define SET_F64_RD(insn, regs, val) \
-	(SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY())
+	(SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY(regs))
 
 #define GET_F32_RS2C(insn, regs) (GET_F32_REG(insn, 2, regs))
 #define GET_F32_RS2S(insn, regs) (GET_F32_REG(RVC_RS2S(insn), 0, regs))

include/sbi/sbi_bitops.h

@@ -12,13 +12,7 @@
 
 #include <sbi/sbi_types.h>
 
-#if __SIZEOF_POINTER__ == 8
-#define BITS_PER_LONG		64
-#elif __SIZEOF_POINTER__ == 4
-#define BITS_PER_LONG		32
-#else
-#error "Unexpected __SIZEOF_POINTER__"
-#endif
+#define BITS_PER_LONG		(8 * __SIZEOF_LONG__)
 
 #define EXTRACT_FIELD(val, which) \
 	(((val) & (which)) / ((which) & ~((which)-1)))
@@ -32,6 +26,7 @@
 #define BIT_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
 #define BIT_WORD(bit)		((bit) / BITS_PER_LONG)
 #define BIT_WORD_OFFSET(bit)	((bit) & (BITS_PER_LONG - 1))
+#define BIT_ALIGN(bit, align)	(((bit) + ((align) - 1)) & ~((align) - 1))
 
 #define GENMASK(h, l) \
 	(((~0UL) - (1UL << (l)) + 1) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
@@ -118,6 +113,22 @@ static inline unsigned long sbi_fls(unsigned long word)
 	return num;
 }
 
+/**
+ * sbi_popcount - find the number of set bit in a long word
+ * @word: the word to search
+ */
+static inline unsigned long sbi_popcount(unsigned long word)
+{
+	unsigned long count = 0;
+
+	while (word) {
+		word &= word - 1;
+		count++;
+	}
+
+	return count;
+}
+
 #define for_each_set_bit(bit, addr, size) \
 	for ((bit) = find_first_bit((addr), (size));		\
 	     (bit) < (size);					\

include/sbi/sbi_byteorder.h

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ */
+
+#ifndef __SBI_BYTEORDER_H__
+#define __SBI_BYTEORDER_H__
+
+#include <sbi/sbi_types.h>
+
+#define BSWAP16(x)	((((x) & 0x00ff) << 8) | \
+			 (((x) & 0xff00) >> 8))
+#define BSWAP32(x)	((((x) & 0x000000ff) << 24) | \
+			 (((x) & 0x0000ff00) << 8) | \
+			 (((x) & 0x00ff0000) >> 8) | \
+			 (((x) & 0xff000000) >> 24))
+#define BSWAP64(x)	((((x) & 0x00000000000000ffULL) << 56) | \
+			 (((x) & 0x000000000000ff00ULL) << 40) | \
+			 (((x) & 0x0000000000ff0000ULL) << 24) | \
+			 (((x) & 0x00000000ff000000ULL) << 8) | \
+			 (((x) & 0x000000ff00000000ULL) >> 8) | \
+			 (((x) & 0x0000ff0000000000ULL) >> 24) | \
+			 (((x) & 0x00ff000000000000ULL) >> 40) | \
+			 (((x) & 0xff00000000000000ULL) >> 56))
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__  /* CPU(little-endian) */
+#define cpu_to_be16(x)		((uint16_t)BSWAP16(x))
+#define cpu_to_be32(x)		((uint32_t)BSWAP32(x))
+#define cpu_to_be64(x)		((uint64_t)BSWAP64(x))
+
+#define be16_to_cpu(x)		((uint16_t)BSWAP16(x))
+#define be32_to_cpu(x)		((uint32_t)BSWAP32(x))
+#define be64_to_cpu(x)		((uint64_t)BSWAP64(x))
+
+#define cpu_to_le16(x)		((uint16_t)(x))
+#define cpu_to_le32(x)		((uint32_t)(x))
+#define cpu_to_le64(x)		((uint64_t)(x))
+
+#define le16_to_cpu(x)		((uint16_t)(x))
+#define le32_to_cpu(x)		((uint32_t)(x))
+#define le64_to_cpu(x)		((uint64_t)(x))
+#else /* CPU(big-endian) */
+#define cpu_to_be16(x)		((uint16_t)(x))
+#define cpu_to_be32(x)		((uint32_t)(x))
+#define cpu_to_be64(x)		((uint64_t)(x))
+
+#define be16_to_cpu(x)		((uint16_t)(x))
+#define be32_to_cpu(x)		((uint32_t)(x))
+#define be64_to_cpu(x)		((uint64_t)(x))
+
+#define cpu_to_le16(x)		((uint16_t)BSWAP16(x))
+#define cpu_to_le32(x)		((uint32_t)BSWAP32(x))
+#define cpu_to_le64(x)		((uint64_t)BSWAP64(x))
+
+#define le16_to_cpu(x)		((uint16_t)BSWAP16(x))
+#define le32_to_cpu(x)		((uint32_t)BSWAP32(x))
+#define le64_to_cpu(x)		((uint64_t)BSWAP64(x))
+#endif
+
+#endif /* __SBI_BYTEORDER_H__ */

include/sbi/sbi_console.h

@@ -19,6 +19,9 @@ struct sbi_console_device {
 	/** Write a character to the console output */
 	void (*console_putc)(char ch);
 
+	/** Write a character string to the console output */
+	unsigned long (*console_puts)(const char *str, unsigned long len);
+
 	/** Read a character from the console input */
 	int (*console_getc)(void);
 };
@@ -33,8 +36,12 @@ void sbi_putc(char ch);
 
 void sbi_puts(const char *str);
 
+unsigned long sbi_nputs(const char *str, unsigned long len);
+
 void sbi_gets(char *s, int maxwidth, char endchar);
 
+unsigned long sbi_ngets(char *str, unsigned long len);
+
 int __printf(2, 3) sbi_sprintf(char *out, const char *format, ...);
 
 int __printf(3, 4) sbi_snprintf(char *out, u32 out_sz, const char *format, ...);

include/sbi/sbi_cppc.h

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ */
+
+#ifndef __SBI_CPPC_H__
+#define __SBI_CPPC_H__
+
+#include <sbi/sbi_types.h>
+
+/** CPPC device */
+struct sbi_cppc_device {
+	/** Name of the CPPC device */
+	char name[32];
+
+	/** probe - returns register width if implemented, 0 otherwise */
+	int (*cppc_probe)(unsigned long reg);
+
+	/** read the cppc register*/
+	int (*cppc_read)(unsigned long reg, uint64_t *val);
+
+	/** write to the cppc register*/
+	int (*cppc_write)(unsigned long reg, uint64_t val);
+};
+
+int sbi_cppc_probe(unsigned long reg);
+int sbi_cppc_read(unsigned long reg, uint64_t *val);
+int sbi_cppc_write(unsigned long reg, uint64_t val);
+
+const struct sbi_cppc_device *sbi_cppc_get_device(void);
+void sbi_cppc_set_device(const struct sbi_cppc_device *dev);
+
+#endif

include/sbi/sbi_domain.h

@@ -36,11 +36,121 @@ struct sbi_domain_memregion {
 	 */
 	unsigned long base;
 	/** Flags representing memory region attributes */
-#define SBI_DOMAIN_MEMREGION_READABLE		(1UL << 0)
-#define SBI_DOMAIN_MEMREGION_WRITEABLE		(1UL << 1)
-#define SBI_DOMAIN_MEMREGION_EXECUTABLE		(1UL << 2)
-#define SBI_DOMAIN_MEMREGION_MMODE		(1UL << 3)
-#define SBI_DOMAIN_MEMREGION_ACCESS_MASK	(0xfUL)
+#define SBI_DOMAIN_MEMREGION_M_READABLE		(1UL << 0)
+#define SBI_DOMAIN_MEMREGION_M_WRITABLE		(1UL << 1)
+#define SBI_DOMAIN_MEMREGION_M_EXECUTABLE	(1UL << 2)
+#define SBI_DOMAIN_MEMREGION_SU_READABLE	(1UL << 3)
+#define SBI_DOMAIN_MEMREGION_SU_WRITABLE	(1UL << 4)
+#define SBI_DOMAIN_MEMREGION_SU_EXECUTABLE	(1UL << 5)
+
+#define SBI_DOMAIN_MEMREGION_ACCESS_MASK	(0x3fUL)
+#define SBI_DOMAIN_MEMREGION_M_ACCESS_MASK	(0x7UL)
+#define SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK	(0x38UL)
+
+#define SBI_DOMAIN_MEMREGION_SU_ACCESS_SHIFT	(3)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_RDONLY		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_READABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SUX_MRX		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE   |	\
+		 SBI_DOMAIN_MEMREGION_M_EXECUTABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SUX_MX		\
+		(SBI_DOMAIN_MEMREGION_M_EXECUTABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE |	\
+		 SBI_DOMAIN_MEMREGION_M_WRITABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_READABLE|	\
+		 SBI_DOMAIN_MEMREGION_SU_WRITABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SUR_MRW		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE |	\
+		 SBI_DOMAIN_MEMREGION_M_WRITABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_READABLE)
+
+	/* Shared read-only region between M and SU mode */
+#define SBI_DOMAIN_MEMREGION_IS_SUR_MR(__flags)			 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_RDONLY)
+
+	/* Shared region: SU execute-only and M read/execute */
+#define SBI_DOMAIN_MEMREGION_IS_SUX_MRX(__flags)		 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SUX_MRX)
+
+	/* Shared region: SU and M execute-only */
+#define SBI_DOMAIN_MEMREGION_IS_SUX_MX(__flags)			 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SUX_MX)
+
+	/* Shared region: SU and M read/write */
+#define SBI_DOMAIN_MEMREGION_IS_SURW_MRW(__flags)		 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW)
+
+	/* Shared region: SU read-only and M read/write */
+#define SBI_DOMAIN_MEMREGION_IS_SUR_MRW(__flags)		 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SUR_MRW)
+
+	/*
+	 * Check if region flags match with any of the above
+	 * mentioned shared region type
+	 */
+#define SBI_DOMAIN_MEMREGION_IS_SHARED(_flags)			\
+		(SBI_DOMAIN_MEMREGION_IS_SUR_MR(_flags)  ||	\
+		 SBI_DOMAIN_MEMREGION_IS_SUX_MRX(_flags) ||	\
+		 SBI_DOMAIN_MEMREGION_IS_SUX_MX(_flags)  ||	\
+		 SBI_DOMAIN_MEMREGION_IS_SURW_MRW(_flags)||	\
+		 SBI_DOMAIN_MEMREGION_IS_SUR_MRW(_flags))
+
+#define SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(__flags)			\
+		((__flags & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK) &&	\
+		 !(__flags & SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK))
+
+#define SBI_DOMAIN_MEMREGION_SU_ONLY_ACCESS(__flags)			\
+		((__flags & SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK)  &&	\
+		 !(__flags & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK))
+
+/** Bit to control if permissions are enforced on all modes */
+#define SBI_DOMAIN_MEMREGION_ENF_PERMISSIONS	(1UL << 6)
+
+#define SBI_DOMAIN_MEMREGION_M_RWX		\
+				(SBI_DOMAIN_MEMREGION_M_READABLE | \
+				 SBI_DOMAIN_MEMREGION_M_WRITABLE | \
+				 SBI_DOMAIN_MEMREGION_M_EXECUTABLE)
+
+#define SBI_DOMAIN_MEMREGION_SU_RWX		\
+				(SBI_DOMAIN_MEMREGION_SU_READABLE | \
+				 SBI_DOMAIN_MEMREGION_SU_WRITABLE | \
+				 SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+
+/* Unrestricted M-mode accesses but enfoced on SU-mode */
+#define SBI_DOMAIN_MEMREGION_READABLE		\
+				(SBI_DOMAIN_MEMREGION_SU_READABLE | \
+				 SBI_DOMAIN_MEMREGION_M_RWX)
+#define SBI_DOMAIN_MEMREGION_WRITEABLE		\
+				(SBI_DOMAIN_MEMREGION_SU_WRITABLE | \
+				 SBI_DOMAIN_MEMREGION_M_RWX)
+#define SBI_DOMAIN_MEMREGION_EXECUTABLE		\
+				(SBI_DOMAIN_MEMREGION_SU_EXECUTABLE | \
+				 SBI_DOMAIN_MEMREGION_M_RWX)
+
+/* Enforced accesses across all modes */
+#define SBI_DOMAIN_MEMREGION_ENF_READABLE	\
+				(SBI_DOMAIN_MEMREGION_SU_READABLE | \
+				 SBI_DOMAIN_MEMREGION_M_READABLE)
+#define SBI_DOMAIN_MEMREGION_ENF_WRITABLE	\
+				(SBI_DOMAIN_MEMREGION_SU_WRITABLE | \
+				 SBI_DOMAIN_MEMREGION_M_WRITABLE)
+#define SBI_DOMAIN_MEMREGION_ENF_EXECUTABLE	\
+				(SBI_DOMAIN_MEMREGION_SU_EXECUTABLE | \
+				 SBI_DOMAIN_MEMREGION_M_EXECUTABLE)
 
 #define SBI_DOMAIN_MEMREGION_MMIO		(1UL << 31)
 	unsigned long flags;
@@ -78,21 +188,21 @@ struct sbi_domain {
 	unsigned long next_mode;
 	/** Is domain allowed to reset the system */
 	bool system_reset_allowed;
+	/** Is domain allowed to suspend the system */
+	bool system_suspend_allowed;
+	/** Identifies whether to include the firmware region */
+	bool fw_region_inited;
 };
 
 /** The root domain instance */
 extern struct sbi_domain root;
 
-/** HART id to domain table */
-extern struct sbi_domain *hartid_to_domain_table[];
-
-/** Get pointer to sbi_domain from HART id */
-#define sbi_hartid_to_domain(__hartid) \
-	hartid_to_domain_table[__hartid]
+/** Get pointer to sbi_domain from HART index */
+struct sbi_domain *sbi_hartindex_to_domain(u32 hartindex);
 
 /** Get pointer to sbi_domain for current HART */
 #define sbi_domain_thishart_ptr() \
-	sbi_hartid_to_domain(current_hartid())
+	sbi_hartindex_to_domain(sbi_hartid_to_hartindex(current_hartid()))
 
 /** Index to domain table */
 extern struct sbi_domain *domidx_to_domain_table[];
@@ -113,7 +223,7 @@ extern struct sbi_domain *domidx_to_domain_table[];
  * Check whether given HART is assigned to specified domain
  * @param dom pointer to domain
  * @param hartid the HART ID
- * @return TRUE if HART is assigned to domain otherwise FALSE
+ * @return true if HART is assigned to domain otherwise false
  */
 bool sbi_domain_is_assigned_hart(const struct sbi_domain *dom, u32 hartid);
 
@@ -148,12 +258,27 @@ void sbi_domain_memregion_init(unsigned long addr,
  * @param addr the address to be checked
  * @param mode the privilege mode of access
  * @param access_flags bitmask of domain access types (enum sbi_domain_access)
- * @return TRUE if access allowed otherwise FALSE
+ * @return true if access allowed otherwise false
  */
 bool sbi_domain_check_addr(const struct sbi_domain *dom,
 			   unsigned long addr, unsigned long mode,
 			   unsigned long access_flags);
 
+/**
+ * Check whether we can access specified address range for given mode and
+ * memory region flags under a domain
+ * @param dom pointer to domain
+ * @param addr the start of the address range to be checked
+ * @param size the size of the address range to be checked
+ * @param mode the privilege mode of access
+ * @param access_flags bitmask of domain access types (enum sbi_domain_access)
+ * @return TRUE if access allowed otherwise FALSE
+ */
+bool sbi_domain_check_addr_range(const struct sbi_domain *dom,
+				 unsigned long addr, unsigned long size,
+				 unsigned long mode,
+				 unsigned long access_flags);
+
 /** Dump domain details on the console */
 void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix);
 

include/sbi/sbi_ecall.h

@@ -13,22 +13,64 @@
 #include <sbi/sbi_types.h>
 #include <sbi/sbi_list.h>
 
-#define SBI_ECALL_VERSION_MAJOR		1
+#define SBI_ECALL_VERSION_MAJOR		2
 #define SBI_ECALL_VERSION_MINOR		0
 #define SBI_OPENSBI_IMPID		1
 
 struct sbi_trap_regs;
 struct sbi_trap_info;
 
+struct sbi_ecall_return {
+	/* Return flag to skip register update */
+	bool skip_regs_update;
+	/* Return value */
+	unsigned long value;
+};
+
 struct sbi_ecall_extension {
+	/* head is used by the extension list */
 	struct sbi_dlist head;
+	/*
+	 * extid_start and extid_end specify the range for this extension. As
+	 * the initial range may be wider than the valid runtime range, the
+	 * register_extensions callback is responsible for narrowing the range
+	 * before other callbacks may be invoked.
+	 */
 	unsigned long extid_start;
 	unsigned long extid_end;
+	/*
+	 * register_extensions
+	 *
+	 * Calls sbi_ecall_register_extension() one or more times to register
+	 * extension ID range(s) which should be handled by this extension.
+	 * More than one sbi_ecall_extension struct and
+	 * sbi_ecall_register_extension() call is necessary when the supported
+	 * extension ID ranges have gaps. Additionally, extension availability
+	 * must be checked before registering, which means, when this callback
+	 * returns, only valid extension IDs from the initial range, which are
+	 * also available, have been registered.
+	 */
+	int (* register_extensions)(void);
+	/*
+	 * probe
+	 *
+	 * Implements the Base extension's probe function for the extension. As
+	 * the register_extensions callback ensures that no other extension
+	 * callbacks will be invoked when the extension is not available, then
+	 * probe can never fail. However, an extension may choose to set
+	 * out_val to a nonzero value other than one. In those cases, it should
+	 * implement this callback.
+	 */
 	int (* probe)(unsigned long extid, unsigned long *out_val);
+	/*
+	 * handle
+	 *
+	 * This is the extension handler. register_extensions ensures it is
+	 * never invoked with an invalid or unavailable extension ID.
+	 */
 	int (* handle)(unsigned long extid, unsigned long funcid,
-		       const struct sbi_trap_regs *regs,
-		       unsigned long *out_val,
-		       struct sbi_trap_info *out_trap);
+		       struct sbi_trap_regs *regs,
+		       struct sbi_ecall_return *out);
 };
 
 u16 sbi_ecall_version_major(void);

include/sbi/sbi_ecall_interface.h

@@ -29,6 +29,9 @@
 #define SBI_EXT_HSM				0x48534D
 #define SBI_EXT_SRST				0x53525354
 #define SBI_EXT_PMU				0x504D55
+#define SBI_EXT_DBCN				0x4442434E
+#define SBI_EXT_SUSP				0x53555350
+#define SBI_EXT_CPPC				0x43505043
 
 /* SBI function IDs for BASE extension*/
 #define SBI_EXT_BASE_GET_SPEC_VERSION		0x0
@@ -99,6 +102,8 @@
 #define SBI_EXT_PMU_COUNTER_START	0x3
 #define SBI_EXT_PMU_COUNTER_STOP	0x4
 #define SBI_EXT_PMU_COUNTER_FW_READ	0x5
+#define SBI_EXT_PMU_COUNTER_FW_READ_HI	0x6
+#define SBI_EXT_PMU_SNAPSHOT_SET_SHMEM	0x7
 
 /** General pmu event codes specified in SBI PMU extension */
 enum sbi_pmu_hw_generic_events_t {
@@ -182,6 +187,17 @@ enum sbi_pmu_fw_event_code_id {
 	SBI_PMU_FW_HFENCE_VVMA_ASID_SENT = 20,
 	SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD = 21,
 	SBI_PMU_FW_MAX,
+	/*
+	 * Event codes 22 to 255 are reserved for future use.
+	 * Event codes 256 to 65534 are reserved for SBI implementation
+	 * specific custom firmware events.
+	 */
+	SBI_PMU_FW_RESERVED_MAX = 0xFFFE,
+	/*
+	 * Event code 0xFFFF is used for platform specific firmware
+	 * events where the event data contains any event specific information.
+	 */
+	SBI_PMU_FW_PLATFORM = 0xFFFF,
 };
 
 /** SBI PMU event idx type */
@@ -200,10 +216,10 @@ enum sbi_pmu_ctr_type {
 };
 
 /* Helper macros to decode event idx */
-#define SBI_PMU_EVENT_IDX_OFFSET 20
 #define SBI_PMU_EVENT_IDX_MASK 0xFFFFF
+#define SBI_PMU_EVENT_IDX_TYPE_OFFSET 16
+#define SBI_PMU_EVENT_IDX_TYPE_MASK (0xF << SBI_PMU_EVENT_IDX_TYPE_OFFSET)
 #define SBI_PMU_EVENT_IDX_CODE_MASK 0xFFFF
-#define SBI_PMU_EVENT_IDX_TYPE_MASK 0xF0000
 #define SBI_PMU_EVENT_RAW_IDX 0x20000
 
 #define SBI_PMU_EVENT_IDX_INVALID 0xFFFFFFFF
@@ -226,9 +242,56 @@ enum sbi_pmu_ctr_type {
 
 /* Flags defined for counter start function */
 #define SBI_PMU_START_FLAG_SET_INIT_VALUE (1 << 0)
+#define SBI_PMU_START_FLAG_INIT_FROM_SNAPSHOT (1 << 1)
 
 /* Flags defined for counter stop function */
 #define SBI_PMU_STOP_FLAG_RESET (1 << 0)
+#define SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT (1 << 1)
+
+/* SBI function IDs for DBCN extension */
+#define SBI_EXT_DBCN_CONSOLE_WRITE		0x0
+#define SBI_EXT_DBCN_CONSOLE_READ		0x1
+#define SBI_EXT_DBCN_CONSOLE_WRITE_BYTE		0x2
+
+/* SBI function IDs for SUSP extension */
+#define SBI_EXT_SUSP_SUSPEND			0x0
+
+#define SBI_SUSP_SLEEP_TYPE_SUSPEND		0x0
+#define SBI_SUSP_SLEEP_TYPE_LAST		SBI_SUSP_SLEEP_TYPE_SUSPEND
+#define SBI_SUSP_PLATFORM_SLEEP_START		0x80000000
+
+/* SBI function IDs for CPPC extension */
+#define SBI_EXT_CPPC_PROBE			0x0
+#define SBI_EXT_CPPC_READ			0x1
+#define SBI_EXT_CPPC_READ_HI			0x2
+#define SBI_EXT_CPPC_WRITE			0x3
+
+enum sbi_cppc_reg_id {
+	SBI_CPPC_HIGHEST_PERF		= 0x00000000,
+	SBI_CPPC_NOMINAL_PERF		= 0x00000001,
+	SBI_CPPC_LOW_NON_LINEAR_PERF	= 0x00000002,
+	SBI_CPPC_LOWEST_PERF		= 0x00000003,
+	SBI_CPPC_GUARANTEED_PERF	= 0x00000004,
+	SBI_CPPC_DESIRED_PERF		= 0x00000005,
+	SBI_CPPC_MIN_PERF		= 0x00000006,
+	SBI_CPPC_MAX_PERF		= 0x00000007,
+	SBI_CPPC_PERF_REDUC_TOLERANCE	= 0x00000008,
+	SBI_CPPC_TIME_WINDOW		= 0x00000009,
+	SBI_CPPC_CTR_WRAP_TIME		= 0x0000000A,
+	SBI_CPPC_REFERENCE_CTR		= 0x0000000B,
+	SBI_CPPC_DELIVERED_CTR		= 0x0000000C,
+	SBI_CPPC_PERF_LIMITED		= 0x0000000D,
+	SBI_CPPC_ENABLE			= 0x0000000E,
+	SBI_CPPC_AUTO_SEL_ENABLE	= 0x0000000F,
+	SBI_CPPC_AUTO_ACT_WINDOW	= 0x00000010,
+	SBI_CPPC_ENERGY_PERF_PREFERENCE	= 0x00000011,
+	SBI_CPPC_REFERENCE_PERF		= 0x00000012,
+	SBI_CPPC_LOWEST_FREQ		= 0x00000013,
+	SBI_CPPC_NOMINAL_FREQ		= 0x00000014,
+	SBI_CPPC_ACPI_LAST		= SBI_CPPC_NOMINAL_FREQ,
+	SBI_CPPC_TRANSITION_LATENCY	= 0x80000000,
+	SBI_CPPC_NON_ACPI_LAST		= SBI_CPPC_TRANSITION_LATENCY,
+};
 
 /* SBI base specification related macros */
 #define SBI_SPEC_VERSION_MAJOR_OFFSET		24
@@ -249,8 +312,9 @@ enum sbi_pmu_ctr_type {
 #define SBI_ERR_ALREADY_AVAILABLE		-6
 #define SBI_ERR_ALREADY_STARTED			-7
 #define SBI_ERR_ALREADY_STOPPED			-8
+#define SBI_ERR_NO_SHMEM			-9
 
-#define SBI_LAST_ERR				SBI_ERR_ALREADY_STOPPED
+#define SBI_LAST_ERR				SBI_ERR_NO_SHMEM
 
 /* clang-format on */
 

include/sbi/sbi_error.h

@@ -23,6 +23,7 @@
 #define SBI_EALREADY		SBI_ERR_ALREADY_AVAILABLE
 #define SBI_EALREADY_STARTED	SBI_ERR_ALREADY_STARTED
 #define SBI_EALREADY_STOPPED	SBI_ERR_ALREADY_STOPPED
+#define SBI_ENO_SHMEM		SBI_ERR_NO_SHMEM
 
 #define SBI_ENODEV		-1000
 #define SBI_ENOSYS		-1001
@@ -31,9 +32,8 @@
 #define SBI_EILL		-1004
 #define SBI_ENOSPC		-1005
 #define SBI_ENOMEM		-1006
-#define SBI_ETRAP		-1007
-#define SBI_EUNKNOWN		-1008
-#define SBI_ENOENT		-1009
+#define SBI_EUNKNOWN		-1007
+#define SBI_ENOENT		-1008
 
 /* clang-format on */
 

include/sbi/sbi_hart.h

@@ -11,6 +11,7 @@
 #define __SBI_HART_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_bitops.h>
 
 /** Possible privileged specification versions of a hart */
 enum sbi_hart_priv_versions {
@@ -26,29 +27,67 @@ enum sbi_hart_priv_versions {
 
 /** Possible ISA extensions of a hart */
 enum sbi_hart_extensions {
-	/** Hart has Sscofpmt extension */
-	SBI_HART_EXT_SSCOFPMF = 0,
-	/** HART has HW time CSR (extension name not available) */
-	SBI_HART_EXT_TIME,
 	/** HART has AIA M-mode CSRs */
-	SBI_HART_EXT_SMAIA,
+	SBI_HART_EXT_SMAIA = 0,
+	/** HART has Smepmp */
+	SBI_HART_EXT_SMEPMP,
 	/** HART has Smstateen CSR **/
 	SBI_HART_EXT_SMSTATEEN,
+	/** Hart has Sscofpmt extension */
+	SBI_HART_EXT_SSCOFPMF,
 	/** HART has Sstc extension */
 	SBI_HART_EXT_SSTC,
+	/** HART has Zicntr extension (i.e. HW cycle, time & instret CSRs) */
+	SBI_HART_EXT_ZICNTR,
+	/** HART has Zihpm extension */
+	SBI_HART_EXT_ZIHPM,
+	/** HART has Zkr extension */
+	SBI_HART_EXT_ZKR,
+	/** Hart has Smcntrpmf extension */
+	SBI_HART_EXT_SMCNTRPMF,
+	/** Hart has Xandespmu extension */
+	SBI_HART_EXT_XANDESPMU,
+	/** Hart has Zicboz extension */
+	SBI_HART_EXT_ZICBOZ,
+	/** Hart has Zicbom extension */
+	SBI_HART_EXT_ZICBOM,
+	/** Hart has Svpbmt extension */
+	SBI_HART_EXT_SVPBMT,
 
 	/** Maximum index of Hart extension */
 	SBI_HART_EXT_MAX,
 };
 
+struct sbi_hart_ext_data {
+	const unsigned int id;
+	const char *name;
+};
+
+extern const struct sbi_hart_ext_data sbi_hart_ext[];
+
+/*
+ * Smepmp enforces access boundaries between M-mode and
+ * S/U-mode. When it is enabled, the PMPs are programmed
+ * such that M-mode doesn't have access to S/U-mode memory.
+ *
+ * To give M-mode R/W access to the shared memory between M and
+ * S/U-mode, first entry is reserved. It is disabled at boot.
+ * When shared memory access is required, the physical address
+ * should be programmed into the first PMP entry with R/W
+ * permissions to the M-mode. Once the work is done, it should be
+ * unmapped. sbi_hart_map_saddr/sbi_hart_unmap_saddr function
+ * pair should be used to map/unmap the shared memory.
+ */
+#define SBI_SMEPMP_RESV_ENTRY		0
+
 struct sbi_hart_features {
 	bool detected;
 	int priv_version;
-	unsigned long extensions;
+	unsigned long extensions[BITS_TO_LONGS(SBI_HART_EXT_MAX)];
 	unsigned int pmp_count;
 	unsigned int pmp_addr_bits;
-	unsigned long pmp_gran;
-	unsigned int mhpm_count;
+	unsigned int pmp_log2gran;
+	unsigned int mhpm_mask;
 	unsigned int mhpm_bits;
 };
 
@@ -63,14 +102,16 @@ static inline ulong sbi_hart_expected_trap_addr(void)
 	return (ulong)sbi_hart_expected_trap;
 }
 
-unsigned int sbi_hart_mhpm_count(struct sbi_scratch *scratch);
+unsigned int sbi_hart_mhpm_mask(struct sbi_scratch *scratch);
 void sbi_hart_delegation_dump(struct sbi_scratch *scratch,
 			      const char *prefix, const char *suffix);
 unsigned int sbi_hart_pmp_count(struct sbi_scratch *scratch);
-unsigned long sbi_hart_pmp_granularity(struct sbi_scratch *scratch);
+unsigned int sbi_hart_pmp_log2gran(struct sbi_scratch *scratch);
 unsigned int sbi_hart_pmp_addrbits(struct sbi_scratch *scratch);
 unsigned int sbi_hart_mhpm_bits(struct sbi_scratch *scratch);
 int sbi_hart_pmp_configure(struct sbi_scratch *scratch);
+int sbi_hart_map_saddr(unsigned long base, unsigned long size);
+int sbi_hart_unmap_saddr(void);
 int sbi_hart_priv_version(struct sbi_scratch *scratch);
 void sbi_hart_get_priv_version_str(struct sbi_scratch *scratch,
 				   char *version_str, int nvstr);

include/sbi/sbi_hartmask.h

@@ -11,6 +11,7 @@
 #define __SBI_HARTMASK_H__
 
 #include <sbi/sbi_bitmap.h>
+#include <sbi/sbi_scratch.h>
 
 /**
  * Maximum number of bits in a hartmask
@@ -32,7 +33,10 @@ struct sbi_hartmask {
 
 /** Initialize hartmask to zero except a particular HART id */
 #define SBI_HARTMASK_INIT_EXCEPT(__m, __h)	\
-	bitmap_zero_except(((__m)->bits), (__h), SBI_HARTMASK_MAX_BITS)
+	do { \
+		u32 __i = sbi_hartid_to_hartindex(__h); \
+		bitmap_zero_except(((__m)->bits), __i, SBI_HARTMASK_MAX_BITS); \
+	} while(0)
 
 /**
  * Get underlying bitmap of hartmask
@@ -41,37 +45,68 @@ struct sbi_hartmask {
 #define sbi_hartmask_bits(__m)		((__m)->bits)
 
 /**
- * Set a HART in hartmask
+ * Set a HART index in hartmask
+ * @param i HART index to set
+ * @param m the hartmask pointer
+ */
+static inline void sbi_hartmask_set_hartindex(u32 i, struct sbi_hartmask *m)
+{
+	if (i < SBI_HARTMASK_MAX_BITS)
+		__set_bit(i, m->bits);
+}
+
+/**
+ * Set a HART id in hartmask
  * @param h HART id to set
  * @param m the hartmask pointer
  */
-static inline void sbi_hartmask_set_hart(u32 h, struct sbi_hartmask *m)
+static inline void sbi_hartmask_set_hartid(u32 h, struct sbi_hartmask *m)
 {
-	if (h < SBI_HARTMASK_MAX_BITS)
-		__set_bit(h, m->bits);
+	sbi_hartmask_set_hartindex(sbi_hartid_to_hartindex(h), m);
 }
 
 /**
- * Clear a HART in hartmask
+ * Clear a HART index in hartmask
+ * @param i HART index to clear
+ * @param m the hartmask pointer
+ */
+static inline void sbi_hartmask_clear_hartindex(u32 i, struct sbi_hartmask *m)
+{
+	if (i < SBI_HARTMASK_MAX_BITS)
+		__clear_bit(i, m->bits);
+}
+
+/**
+ * Clear a HART id in hartmask
  * @param h HART id to clear
  * @param m the hartmask pointer
  */
-static inline void sbi_hartmask_clear_hart(u32 h, struct sbi_hartmask *m)
+static inline void sbi_hartmask_clear_hartid(u32 h, struct sbi_hartmask *m)
 {
-	if (h < SBI_HARTMASK_MAX_BITS)
-		__clear_bit(h, m->bits);
+	sbi_hartmask_clear_hartindex(sbi_hartid_to_hartindex(h), m);
 }
 
 /**
- * Test a HART in hartmask
+ * Test a HART index in hartmask
+ * @param i HART index to test
+ * @param m the hartmask pointer
+ */
+static inline int sbi_hartmask_test_hartindex(u32 i,
+					      const struct sbi_hartmask *m)
+{
+	if (i < SBI_HARTMASK_MAX_BITS)
+		return __test_bit(i, m->bits);
+	return 0;
+}
+
+/**
+ * Test a HART id in hartmask
  * @param h HART id to test
  * @param m the hartmask pointer
  */
-static inline int sbi_hartmask_test_hart(u32 h, const struct sbi_hartmask *m)
+static inline int sbi_hartmask_test_hartid(u32 h, const struct sbi_hartmask *m)
 {
-	if (h < SBI_HARTMASK_MAX_BITS)
-		return __test_bit(h, m->bits);
-	return 0;
+	return sbi_hartmask_test_hartindex(sbi_hartid_to_hartindex(h), m);
 }
 
 /**
@@ -134,8 +169,14 @@ static inline void sbi_hartmask_xor(struct sbi_hartmask *dstp,
 		   sbi_hartmask_bits(src2p), SBI_HARTMASK_MAX_BITS);
 }
 
-/** Iterate over each HART in hartmask */
-#define sbi_hartmask_for_each_hart(__h, __m)	\
-	for_each_set_bit(__h, (__m)->bits, SBI_HARTMASK_MAX_BITS)
+/**
+ * Iterate over each HART index in hartmask
+ * __i hart index
+ * __m hartmask
+*/
+#define sbi_hartmask_for_each_hartindex(__i, __m) \
+	for((__i) = find_first_bit((__m)->bits, SBI_HARTMASK_MAX_BITS); \
+		(__i) < SBI_HARTMASK_MAX_BITS; \
+		(__i) = find_next_bit((__m)->bits, SBI_HARTMASK_MAX_BITS, (__i) + 1))
 
 #endif

include/sbi/sbi_heap.h

@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel<apatel@ventanamicro.com>
+ */
+
+#ifndef __SBI_HEAP_H__
+#define __SBI_HEAP_H__
+
+#include <sbi/sbi_types.h>
+
+/* Alignment of heap base address and size */
+#define HEAP_BASE_ALIGN			1024
+
+struct sbi_scratch;
+
+/** Allocate from heap area */
+void *sbi_malloc(size_t size);
+
+/** Zero allocate from heap area */
+void *sbi_zalloc(size_t size);
+
+/** Allocate array from heap area */
+static inline void *sbi_calloc(size_t nitems, size_t size)
+{
+	return sbi_zalloc(nitems * size);
+}
+
+/** Free-up to heap area */
+void sbi_free(void *ptr);
+
+/** Amount (in bytes) of free space in the heap area */
+unsigned long sbi_heap_free_space(void);
+
+/** Amount (in bytes) of used space in the heap area */
+unsigned long sbi_heap_used_space(void);
+
+/** Amount (in bytes) of reserved space in the heap area */
+unsigned long sbi_heap_reserved_space(void);
+
+/** Initialize heap area */
+int sbi_heap_init(struct sbi_scratch *scratch);
+
+#endif

include/sbi/sbi_hsm.h

@@ -21,8 +21,12 @@ struct sbi_hsm_device {
 	int (*hart_start)(u32 hartid, ulong saddr);
 
 	/**
-	 * Stop (or power-down) the current hart from running. This call
-	 * doesn't expect to return if success.
+	 * Stop (or power-down) the current hart from running.
+	 *
+	 * Return SBI_ENOTSUPP if the hart does not support platform-specific
+	 * stop actions.
+	 *
+	 * For successful stop, the call won't return.
 	 */
 	int (*hart_stop)(void);
 
@@ -59,15 +63,21 @@ void __noreturn sbi_hsm_exit(struct sbi_scratch *scratch);
 
 int sbi_hsm_hart_start(struct sbi_scratch *scratch,
 		       const struct sbi_domain *dom,
-		       u32 hartid, ulong saddr, ulong smode, ulong priv);
+		       u32 hartid, ulong saddr, ulong smode, ulong arg1);
 int sbi_hsm_hart_stop(struct sbi_scratch *scratch, bool exitnow);
 void sbi_hsm_hart_resume_start(struct sbi_scratch *scratch);
-void sbi_hsm_hart_resume_finish(struct sbi_scratch *scratch);
+void __noreturn sbi_hsm_hart_resume_finish(struct sbi_scratch *scratch,
+					   u32 hartid);
 int sbi_hsm_hart_suspend(struct sbi_scratch *scratch, u32 suspend_type,
-			 ulong raddr, ulong rmode, ulong priv);
+			 ulong raddr, ulong rmode, ulong arg1);
+bool sbi_hsm_hart_change_state(struct sbi_scratch *scratch, long oldstate,
+			       long newstate);
+int __sbi_hsm_hart_get_state(u32 hartid);
 int sbi_hsm_hart_get_state(const struct sbi_domain *dom, u32 hartid);
 int sbi_hsm_hart_interruptible_mask(const struct sbi_domain *dom,
 				    ulong hbase, ulong *out_hmask);
-void sbi_hsm_prepare_next_jump(struct sbi_scratch *scratch, u32 hartid);
+void __sbi_hsm_suspend_non_ret_save(struct sbi_scratch *scratch);
+void __noreturn sbi_hsm_hart_start_finish(struct sbi_scratch *scratch,
+					  u32 hartid);
 
 #endif

include/sbi/sbi_init.h

@@ -16,6 +16,8 @@ struct sbi_scratch;
 
 void __noreturn sbi_init(struct sbi_scratch *scratch);
 
+unsigned long sbi_entry_count(u32 hartid);
+
 unsigned long sbi_init_count(u32 hartid);
 
 void __noreturn sbi_exit(struct sbi_scratch *scratch);

include/sbi/sbi_ipi.h

@@ -14,7 +14,7 @@
 
 /* clang-format off */
 
-#define SBI_IPI_EVENT_MAX			__riscv_xlen
+#define SBI_IPI_EVENT_MAX			(8 * __SIZEOF_LONG__)
 
 /* clang-format on */
 
@@ -23,11 +23,17 @@ struct sbi_ipi_device {
 	/** Name of the IPI device */
 	char name[32];
 
-	/** Send IPI to a target HART */
-	void (*ipi_send)(u32 target_hart);
+	/** Send IPI to a target HART index */
+	void (*ipi_send)(u32 hart_index);
 
-	/** Clear IPI for a target HART */
-	void (*ipi_clear)(u32 target_hart);
+	/** Clear IPI for a target HART index */
+	void (*ipi_clear)(u32 hart_index);
+};
+
+enum sbi_ipi_update_type {
+	SBI_IPI_UPDATE_SUCCESS,
+	SBI_IPI_UPDATE_BREAK,
+	SBI_IPI_UPDATE_RETRY,
 };
 
 struct sbi_scratch;
@@ -41,10 +47,14 @@ struct sbi_ipi_event_ops {
 	 * Update callback to save/enqueue data for remote HART
 	 * Note: This is an optional callback and it is called just before
 	 * triggering IPI to remote HART.
+	 * @return < 0, error or failure
+	 * @return SBI_IPI_UPDATE_SUCCESS, success
+	 * @return SBI_IPI_UPDATE_BREAK, break IPI, done on local hart
+	 * @return SBI_IPI_UPDATE_RETRY, need retry
 	 */
 	int (* update)(struct sbi_scratch *scratch,
 			struct sbi_scratch *remote_scratch,
-			u32 remote_hartid, void *data);
+			u32 remote_hartindex, void *data);
 
 	/**
 	 * Sync callback to wait for remote HART
@@ -75,7 +85,9 @@ int sbi_ipi_send_halt(ulong hmask, ulong hbase);
 
 void sbi_ipi_process(void);
 
-int sbi_ipi_raw_send(u32 target_hart);
+int sbi_ipi_raw_send(u32 hartindex);
+
+void sbi_ipi_raw_clear(u32 hartindex);
 
 const struct sbi_ipi_device *sbi_ipi_get_device(void);
 

include/sbi/sbi_list.h

@@ -31,7 +31,7 @@ struct sbi_dlist _lname = SBI_LIST_HEAD_INIT(_lname)
 #define SBI_INIT_LIST_HEAD(ptr)	\
 do { \
 	(ptr)->next = ptr; (ptr)->prev = ptr; \
-} while (0);
+} while (0)
 
 static inline void __sbi_list_add(struct sbi_dlist *new,
 				  struct sbi_dlist *prev,
@@ -47,7 +47,7 @@ static inline void __sbi_list_add(struct sbi_dlist *new,
  * Checks if the list is empty or not.
  * @param head List head
  *
- * Retruns TRUE if list is empty, FALSE otherwise.
+ * Returns true if list is empty, false otherwise.
  */
 static inline bool sbi_list_empty(struct sbi_dlist *head)
 {

include/sbi/sbi_platform.h

@@ -29,12 +29,16 @@
 #define SBI_PLATFORM_HART_COUNT_OFFSET (0x50)
 /** Offset of hart_stack_size in struct sbi_platform */
 #define SBI_PLATFORM_HART_STACK_SIZE_OFFSET (0x54)
+/** Offset of heap_size in struct sbi_platform */
+#define SBI_PLATFORM_HEAP_SIZE_OFFSET (0x58)
+/** Offset of reserved in struct sbi_platform */
+#define SBI_PLATFORM_RESERVED_OFFSET (0x5c)
 /** Offset of platform_ops_addr in struct sbi_platform */
-#define SBI_PLATFORM_OPS_OFFSET (0x58)
+#define SBI_PLATFORM_OPS_OFFSET (0x60)
 /** Offset of firmware_context in struct sbi_platform */
-#define SBI_PLATFORM_FIRMWARE_CONTEXT_OFFSET (0x58 + __SIZEOF_POINTER__)
+#define SBI_PLATFORM_FIRMWARE_CONTEXT_OFFSET (0x60 + __SIZEOF_POINTER__)
 /** Offset of hart_index2id in struct sbi_platform */
-#define SBI_PLATFORM_HART_INDEX2ID_OFFSET (0x58 + (__SIZEOF_POINTER__ * 2))
+#define SBI_PLATFORM_HART_INDEX2ID_OFFSET (0x60 + (__SIZEOF_POINTER__ * 2))
 
 #define SBI_PLATFORM_TLB_RANGE_FLUSH_LIMIT_DEFAULT		(1UL << 12)
 
@@ -46,7 +50,7 @@
 #include <sbi/sbi_version.h>
 
 struct sbi_domain_memregion;
-struct sbi_trap_info;
+struct sbi_ecall_return;
 struct sbi_trap_regs;
 struct sbi_hart_features;
 
@@ -65,6 +69,9 @@ enum sbi_platform_features {
 
 /** Platform functions */
 struct sbi_platform_operations {
+	/* Check if specified HART is allowed to do cold boot */
+	bool (*cold_boot_allowed)(u32 hartid);
+
 	/* Platform nascent initialization */
 	int (*nascent_init)(void);
 
@@ -118,23 +125,29 @@ struct sbi_platform_operations {
 	/** Get tlb flush limit value **/
 	u64 (*get_tlbr_flush_limit)(void);
 
+	/** Get tlb fifo num entries*/
+	u32 (*get_tlb_num_entries)(void);
+
 	/** Initialize platform timer for current HART */
 	int (*timer_init)(bool cold_boot);
 	/** Exit platform timer for current HART */
 	void (*timer_exit)(void);
 
-	/** platform specific SBI extension implementation probe function */
-	int (*vendor_ext_check)(long extid);
+	/** Check if SBI vendor extension is implemented or not */
+	bool (*vendor_ext_check)(void);
 	/** platform specific SBI extension implementation provider */
-	int (*vendor_ext_provider)(long extid, long funcid,
-				   const struct sbi_trap_regs *regs,
-				   unsigned long *out_value,
-				   struct sbi_trap_info *out_trap);
+	int (*vendor_ext_provider)(long funcid,
+				   struct sbi_trap_regs *regs,
+				   struct sbi_ecall_return *out);
 };
 
 /** Platform default per-HART stack size for exception/interrupt handling */
 #define SBI_PLATFORM_DEFAULT_HART_STACK_SIZE	8192
 
+/** Platform default heap size */
+#define SBI_PLATFORM_DEFAULT_HEAP_SIZE(__num_hart)	\
+					(0x8000 + 0x800 * (__num_hart))
+
 /** Representation of a platform */
 struct sbi_platform {
 	/**
@@ -157,6 +170,10 @@ struct sbi_platform {
 	u32 hart_count;
 	/** Per-HART stack size for exception/interrupt handling */
 	u32 hart_stack_size;
+	/** Size of heap shared by all HARTs */
+	u32 heap_size;
+	/** Reserved for future use */
+	u32 reserved;
 	/** Pointer to sbi platform operations */
 	unsigned long platform_ops_addr;
 	/** Pointer to system firmware specific context */
@@ -243,16 +260,6 @@ _Static_assert(
 #define sbi_platform_has_mfaults_delegation(__p) \
 	((__p)->features & SBI_PLATFORM_HAS_MFAULTS_DELEGATION)
 
-/**
- * Get HART index for the given HART
- *
- * @param plat pointer to struct sbi_platform
- * @param hartid HART ID
- *
- * @return 0 <= value < hart_count for valid HART otherwise -1U
- */
-u32 sbi_platform_hart_index(const struct sbi_platform *plat, u32 hartid);
-
 /**
  * Get the platform features in string format
  *
@@ -310,6 +317,20 @@ static inline u64 sbi_platform_tlbr_flush_limit(const struct sbi_platform *plat)
 	return SBI_PLATFORM_TLB_RANGE_FLUSH_LIMIT_DEFAULT;
 }
 
+/**
+ * Get platform specific tlb fifo num entries.
+ *
+ * @param plat pointer to struct sbi_platform
+ *
+ * @return number of tlb fifo entries
+*/
+static inline u32 sbi_platform_tlb_fifo_num_entries(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->get_tlb_num_entries)
+		return sbi_platform_ops(plat)->get_tlb_num_entries();
+	return sbi_scratch_last_hartindex() + 1;
+}
+
 /**
  * Get total number of HARTs supported by the platform
  *
@@ -339,21 +360,20 @@ static inline u32 sbi_platform_hart_stack_size(const struct sbi_platform *plat)
 }
 
 /**
- * Check whether given HART is invalid
+ * Check whether given HART is allowed to do cold boot
  *
  * @param plat pointer to struct sbi_platform
  * @param hartid HART ID
  *
- * @return TRUE if HART is invalid and FALSE otherwise
+ * @return true if HART is allowed to do cold boot and false otherwise
  */
-static inline bool sbi_platform_hart_invalid(const struct sbi_platform *plat,
-					     u32 hartid)
+static inline bool sbi_platform_cold_boot_allowed(
+					const struct sbi_platform *plat,
+					u32 hartid)
 {
-	if (!plat)
-		return TRUE;
-	if (plat->hart_count <= sbi_platform_hart_index(plat, hartid))
-		return TRUE;
-	return FALSE;
+	if (plat && sbi_platform_ops(plat)->cold_boot_allowed)
+		return sbi_platform_ops(plat)->cold_boot_allowed(hartid);
+	return true;
 }
 
 /**
@@ -377,7 +397,7 @@ static inline int sbi_platform_nascent_init(const struct sbi_platform *plat)
  * Early initialization for current HART
  *
  * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (TRUE) or warm_boot (FALSE)
+ * @param cold_boot whether cold boot (true) or warm_boot (false)
  *
  * @return 0 on success and negative error code on failure
  */
@@ -393,7 +413,7 @@ static inline int sbi_platform_early_init(const struct sbi_platform *plat,
  * Final initialization for current HART
  *
  * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (TRUE) or warm_boot (FALSE)
+ * @param cold_boot whether cold boot (true) or warm_boot (false)
  *
  * @return 0 on success and negative error code on failure
  */
@@ -538,7 +558,7 @@ static inline int sbi_platform_console_init(const struct sbi_platform *plat)
  * Initialize the platform interrupt controller for current HART
  *
  * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (TRUE) or warm_boot (FALSE)
+ * @param cold_boot whether cold boot (true) or warm_boot (false)
  *
  * @return 0 on success and negative error code on failure
  */
@@ -565,7 +585,7 @@ static inline void sbi_platform_irqchip_exit(const struct sbi_platform *plat)
  * Initialize the platform IPI support for current HART
  *
  * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (TRUE) or warm_boot (FALSE)
+ * @param cold_boot whether cold boot (true) or warm_boot (false)
  *
  * @return 0 on success and negative error code on failure
  */
@@ -592,7 +612,7 @@ static inline void sbi_platform_ipi_exit(const struct sbi_platform *plat)
  * Initialize the platform timer for current HART
  *
  * @param plat pointer to struct sbi_platform
- * @param cold_boot whether cold boot (TRUE) or warm_boot (FALSE)
+ * @param cold_boot whether cold boot (true) or warm_boot (false)
  *
  * @return 0 on success and negative error code on failure
  */
@@ -616,27 +636,25 @@ static inline void sbi_platform_timer_exit(const struct sbi_platform *plat)
 }
 
 /**
- * Check if a vendor extension is implemented or not.
+ * Check if SBI vendor extension is implemented or not.
  *
  * @param plat pointer to struct sbi_platform
- * @param extid	vendor SBI extension id
  *
- * @return 0 if extid is not implemented and 1 if implemented
+ * @return false if not implemented and true if implemented
  */
-static inline int sbi_platform_vendor_ext_check(const struct sbi_platform *plat,
-						long extid)
+static inline bool sbi_platform_vendor_ext_check(
+					const struct sbi_platform *plat)
 {
 	if (plat && sbi_platform_ops(plat)->vendor_ext_check)
-		return sbi_platform_ops(plat)->vendor_ext_check(extid);
+		return sbi_platform_ops(plat)->vendor_ext_check();
 
-	return 0;
+	return false;
 }
 
 /**
  * Invoke platform specific vendor SBI extension implementation.
  *
  * @param plat pointer to struct sbi_platform
- * @param extid	vendor SBI extension id
  * @param funcid SBI function id within the extension id
  * @param regs pointer to trap registers passed by the caller
  * @param out_value output value that can be filled by the callee
@@ -646,17 +664,13 @@ static inline int sbi_platform_vendor_ext_check(const struct sbi_platform *plat,
  */
 static inline int sbi_platform_vendor_ext_provider(
 					const struct sbi_platform *plat,
-					long extid, long funcid,
-					const struct sbi_trap_regs *regs,
-					unsigned long *out_value,
-					struct sbi_trap_info *out_trap)
+					long funcid,
+					struct sbi_trap_regs *regs,
+					struct sbi_ecall_return *out)
 {
-	if (plat && sbi_platform_ops(plat)->vendor_ext_provider) {
-		return sbi_platform_ops(plat)->vendor_ext_provider(extid,
-								funcid, regs,
-								out_value,
-								out_trap);
-	}
+	if (plat && sbi_platform_ops(plat)->vendor_ext_provider)
+		return sbi_platform_ops(plat)->vendor_ext_provider(funcid,
+								regs, out);
 
 	return SBI_ENOTSUPP;
 }

include/sbi/sbi_pmu.h

@@ -23,6 +23,7 @@ struct sbi_scratch;
 #define SBI_PMU_HW_CTR_MAX 32
 #define SBI_PMU_CTR_MAX	   (SBI_PMU_HW_CTR_MAX + SBI_PMU_FW_CTR_MAX)
 #define SBI_PMU_FIXED_CTR_MASK 0x07
+#define SBI_PMU_CY_IR_MASK	0x05
 
 struct sbi_pmu_device {
 	/** Name of the PMU platform device */
@@ -30,37 +31,48 @@ struct sbi_pmu_device {
 
 	/**
 	 * Validate event code of custom firmware event
-	 * Note: SBI_PMU_FW_MAX <= event_idx_code
 	 */
-	int (*fw_event_validate_code)(uint32_t event_idx_code);
+	int (*fw_event_validate_encoding)(uint32_t hartid, uint64_t event_data);
 
 	/**
 	 * Match custom firmware counter with custom firmware event
 	 * Note: 0 <= counter_index < SBI_PMU_FW_CTR_MAX
 	 */
-	bool (*fw_counter_match_code)(uint32_t counter_index,
-				      uint32_t event_idx_code);
+	bool (*fw_counter_match_encoding)(uint32_t hartid,
+					  uint32_t counter_index,
+					  uint64_t event_data);
+
+	/**
+	 * Fetch the max width of this counter in number of bits.
+	 */
+	int (*fw_counter_width)(void);
 
 	/**
 	 * Read value of custom firmware counter
 	 * Note: 0 <= counter_index < SBI_PMU_FW_CTR_MAX
 	 */
-	uint64_t (*fw_counter_read_value)(uint32_t counter_index);
+	uint64_t (*fw_counter_read_value)(uint32_t hartid,
+					  uint32_t counter_index);
+
+	/**
+	 * Write value to custom firmware counter
+	 * Note: 0 <= counter_index < SBI_PMU_FW_CTR_MAX
+	 */
+	void (*fw_counter_write_value)(uint32_t hartid, uint32_t counter_index,
+				       uint64_t value);
 
 	/**
 	 * Start custom firmware counter
-	 * Note: SBI_PMU_FW_MAX <= event_idx_code
 	 * Note: 0 <= counter_index < SBI_PMU_FW_CTR_MAX
 	 */
-	int (*fw_counter_start)(uint32_t counter_index,
-				uint32_t event_idx_code,
-				uint64_t init_val, bool init_val_update);
+	int (*fw_counter_start)(uint32_t hartid, uint32_t counter_index,
+				uint64_t event_data);
 
 	/**
 	 * Stop custom firmware counter
 	 * Note: 0 <= counter_index < SBI_PMU_FW_CTR_MAX
 	 */
-	int (*fw_counter_stop)(uint32_t counter_index);
+	int (*fw_counter_stop)(uint32_t hartid, uint32_t counter_index);
 
 	/**
 	 * Custom enable irq for hardware counter
@@ -78,6 +90,12 @@ struct sbi_pmu_device {
 	 * Custom function returning the machine-specific irq-bit.
 	 */
 	int (*hw_counter_irq_bit)(void);
+
+	/**
+	 * Custom function to inhibit counting of events while in
+	 * specified mode.
+	 */
+	void (*hw_counter_filter_mode)(unsigned long flags, int counter_index);
 };
 
 /** Get the PMU platform device */

include/sbi/sbi_scratch.h

@@ -18,26 +18,32 @@
 #define SBI_SCRATCH_FW_START_OFFSET		(0 * __SIZEOF_POINTER__)
 /** Offset of fw_size member in sbi_scratch */
 #define SBI_SCRATCH_FW_SIZE_OFFSET		(1 * __SIZEOF_POINTER__)
+/** Offset (in sbi_scratch) of the R/W Offset */
+#define SBI_SCRATCH_FW_RW_OFFSET		(2 * __SIZEOF_POINTER__)
+/** Offset of fw_heap_offset member in sbi_scratch */
+#define SBI_SCRATCH_FW_HEAP_OFFSET		(3 * __SIZEOF_POINTER__)
+/** Offset of fw_heap_size_offset member in sbi_scratch */
+#define SBI_SCRATCH_FW_HEAP_SIZE_OFFSET		(4 * __SIZEOF_POINTER__)
 /** Offset of next_arg1 member in sbi_scratch */
-#define SBI_SCRATCH_NEXT_ARG1_OFFSET		(2 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_NEXT_ARG1_OFFSET		(5 * __SIZEOF_POINTER__)
 /** Offset of next_addr member in sbi_scratch */
-#define SBI_SCRATCH_NEXT_ADDR_OFFSET		(3 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_NEXT_ADDR_OFFSET		(6 * __SIZEOF_POINTER__)
 /** Offset of next_mode member in sbi_scratch */
-#define SBI_SCRATCH_NEXT_MODE_OFFSET		(4 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_NEXT_MODE_OFFSET		(7 * __SIZEOF_POINTER__)
 /** Offset of warmboot_addr member in sbi_scratch */
-#define SBI_SCRATCH_WARMBOOT_ADDR_OFFSET	(5 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_WARMBOOT_ADDR_OFFSET	(8 * __SIZEOF_POINTER__)
 /** Offset of platform_addr member in sbi_scratch */
-#define SBI_SCRATCH_PLATFORM_ADDR_OFFSET	(6 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_PLATFORM_ADDR_OFFSET	(9 * __SIZEOF_POINTER__)
 /** Offset of hartid_to_scratch member in sbi_scratch */
-#define SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET	(7 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET	(10 * __SIZEOF_POINTER__)
 /** Offset of trap_exit member in sbi_scratch */
-#define SBI_SCRATCH_TRAP_EXIT_OFFSET		(8 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_TRAP_EXIT_OFFSET		(11 * __SIZEOF_POINTER__)
 /** Offset of tmp0 member in sbi_scratch */
-#define SBI_SCRATCH_TMP0_OFFSET			(9 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_TMP0_OFFSET			(12 * __SIZEOF_POINTER__)
 /** Offset of options member in sbi_scratch */
-#define SBI_SCRATCH_OPTIONS_OFFSET		(10 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_OPTIONS_OFFSET		(13 * __SIZEOF_POINTER__)
 /** Offset of extra space in sbi_scratch */
-#define SBI_SCRATCH_EXTRA_SPACE_OFFSET		(11 * __SIZEOF_POINTER__)
+#define SBI_SCRATCH_EXTRA_SPACE_OFFSET		(14 * __SIZEOF_POINTER__)
 /** Maximum size of sbi_scratch (4KB) */
 #define SBI_SCRATCH_SIZE			(0x1000)
 
@@ -53,6 +59,12 @@ struct sbi_scratch {
 	unsigned long fw_start;
 	/** Size (in bytes) of firmware linked to OpenSBI library */
 	unsigned long fw_size;
+	/** Offset (in bytes) of the R/W section */
+	unsigned long fw_rw_offset;
+	/** Offset (in bytes) of the heap area */
+	unsigned long fw_heap_offset;
+	/** Size (in bytes) of the heap area */
+	unsigned long fw_heap_size;
 	/** Arg1 (or 'a1' register) of next booting stage for this HART */
 	unsigned long next_arg1;
 	/** Address of next booting stage for this HART */
@@ -163,6 +175,9 @@ unsigned long sbi_scratch_alloc_offset(unsigned long size);
 /** Free-up extra space in sbi_scratch */
 void sbi_scratch_free_offset(unsigned long offset);
 
+/** Amount (in bytes) of used space in in sbi_scratch */
+unsigned long sbi_scratch_used_space(void);
+
 /** Get pointer from offset in sbi_scratch */
 #define sbi_scratch_offset_ptr(scratch, offset)	(void *)((char *)(scratch) + (offset))
 
@@ -170,18 +185,68 @@ void sbi_scratch_free_offset(unsigned long offset);
 #define sbi_scratch_thishart_offset_ptr(offset)	\
 	(void *)((char *)sbi_scratch_thishart_ptr() + (offset))
 
-/** HART id to scratch table */
-extern struct sbi_scratch *hartid_to_scratch_table[];
+/** Allocate offset for a data type in sbi_scratch */
+#define sbi_scratch_alloc_type_offset(__type)				\
+	sbi_scratch_alloc_offset(sizeof(__type))
+
+/** Read a data type from sbi_scratch at given offset */
+#define sbi_scratch_read_type(__scratch, __type, __offset)		\
+({									\
+	*((__type *)sbi_scratch_offset_ptr((__scratch), (__offset)));	\
+})
+
+/** Write a data type to sbi_scratch at given offset */
+#define sbi_scratch_write_type(__scratch, __type, __offset, __ptr)	\
+do {									\
+	*((__type *)sbi_scratch_offset_ptr((__scratch), (__offset)))	\
+					= (__type)(__ptr);		\
+} while (0)
+
+/** Last HART index having a sbi_scratch pointer */
+extern u32 last_hartindex_having_scratch;
+
+/** Get last HART index having a sbi_scratch pointer */
+#define sbi_scratch_last_hartindex()	last_hartindex_having_scratch
+
+/** Check whether a particular HART index is valid or not */
+#define sbi_hartindex_valid(__hartindex) \
+(((__hartindex) <= sbi_scratch_last_hartindex()) ? true : false)
+
+/** HART index to HART id table */
+extern u32 hartindex_to_hartid_table[];
+
+/** Get sbi_scratch from HART index */
+#define sbi_hartindex_to_hartid(__hartindex)		\
+({							\
+	((__hartindex) <= sbi_scratch_last_hartindex()) ?\
+	hartindex_to_hartid_table[__hartindex] : -1U;	\
+})
+
+/** HART index to scratch table */
+extern struct sbi_scratch *hartindex_to_scratch_table[];
+
+/** Get sbi_scratch from HART index */
+#define sbi_hartindex_to_scratch(__hartindex)		\
+({							\
+	((__hartindex) <= sbi_scratch_last_hartindex()) ?\
+	hartindex_to_scratch_table[__hartindex] : NULL;\
+})
+
+/**
+ * Get logical index for given HART id
+ * @param hartid physical HART id
+ * @returns value between 0 to SBI_HARTMASK_MAX_BITS upon success and
+ *	    SBI_HARTMASK_MAX_BITS upon failure.
+ */
+u32 sbi_hartid_to_hartindex(u32 hartid);
 
 /** Get sbi_scratch from HART id */
 #define sbi_hartid_to_scratch(__hartid) \
-	hartid_to_scratch_table[__hartid]
+	sbi_hartindex_to_scratch(sbi_hartid_to_hartindex(__hartid))
 
-/** Last HART id having a sbi_scratch pointer */
-extern u32 last_hartid_having_scratch;
-
-/** Get last HART id having a sbi_scratch pointer */
-#define sbi_scratch_last_hartid()	last_hartid_having_scratch
+/** Check whether particular HART id is valid or not */
+#define sbi_hartid_valid(__hartid)	\
+	sbi_hartindex_valid(sbi_hartid_to_hartindex(__hartid))
 
 #endif
 

include/sbi/sbi_system.h

@@ -43,4 +43,38 @@ bool sbi_system_reset_supported(u32 reset_type, u32 reset_reason);
 
 void __noreturn sbi_system_reset(u32 reset_type, u32 reset_reason);
 
+/** System suspend device */
+struct sbi_system_suspend_device {
+	/** Name of the system suspend device */
+	char name[32];
+
+	/**
+	 * Check whether sleep type is supported by the device
+	 *
+	 * Returns 0 when @sleep_type supported, SBI_ERR_INVALID_PARAM
+	 * when @sleep_type is reserved, or SBI_ERR_NOT_SUPPORTED when
+	 * @sleep_type is not reserved and is implemented, but the
+	 * platform doesn't support it due to missing dependencies.
+	 */
+	int (*system_suspend_check)(u32 sleep_type);
+
+	/**
+	 * Suspend the system
+	 *
+	 * @sleep_type: The sleep type identifier passed to the SBI call.
+	 * @mmode_resume_addr:
+	 *     This is the same as sbi_scratch.warmboot_addr. Some platforms
+	 *     may not be able to return from system_suspend(), so they will
+	 *     jump directly to this address instead. Platforms which can
+	 *     return from system_suspend() may ignore this parameter.
+	 */
+	int (*system_suspend)(u32 sleep_type, unsigned long mmode_resume_addr);
+};
+
+const struct sbi_system_suspend_device *sbi_system_suspend_get_device(void);
+void sbi_system_suspend_set_device(struct sbi_system_suspend_device *dev);
+void sbi_system_suspend_test_enable(void);
+bool sbi_system_suspend_supported(u32 sleep_type);
+int sbi_system_suspend(u32 sleep_type, ulong resume_addr, ulong opaque);
+
 #endif

include/sbi/sbi_tlb.h

@@ -20,34 +20,35 @@
 
 /* clang-format on */
 
-#define SBI_TLB_FIFO_NUM_ENTRIES		8
-
 struct sbi_scratch;
 
+enum sbi_tlb_type {
+	SBI_TLB_FENCE_I = 0,
+	SBI_TLB_SFENCE_VMA,
+	SBI_TLB_SFENCE_VMA_ASID,
+	SBI_TLB_HFENCE_GVMA_VMID,
+	SBI_TLB_HFENCE_GVMA,
+	SBI_TLB_HFENCE_VVMA_ASID,
+	SBI_TLB_HFENCE_VVMA,
+	SBI_TLB_TYPE_MAX,
+};
+
 struct sbi_tlb_info {
 	unsigned long start;
 	unsigned long size;
-	unsigned long asid;
-	unsigned long vmid;
-	void (*local_fn)(struct sbi_tlb_info *tinfo);
+	uint16_t asid;
+	uint16_t vmid;
+	enum sbi_tlb_type type;
 	struct sbi_hartmask smask;
 };
 
-void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo);
-
-#define SBI_TLB_INFO_INIT(__p, __start, __size, __asid, __vmid, __lfn, __src) \
+#define SBI_TLB_INFO_INIT(__p, __start, __size, __asid, __vmid, __type, __src) \
 do { \
 	(__p)->start = (__start); \
 	(__p)->size = (__size); \
 	(__p)->asid = (__asid); \
 	(__p)->vmid = (__vmid); \
-	(__p)->local_fn = (__lfn); \
+	(__p)->type = (__type); \
 	SBI_HARTMASK_INIT_EXCEPT(&(__p)->smask, (__src)); \
 } while (0)
 

include/sbi/sbi_types.h

@@ -54,16 +54,22 @@ typedef unsigned long		virtual_size_t;
 typedef unsigned long		physical_addr_t;
 typedef unsigned long		physical_size_t;
 
-#define TRUE			1
-#define FALSE			0
-#define true			TRUE
-#define false			FALSE
+typedef uint16_t		le16_t;
+typedef uint16_t		be16_t;
+typedef uint32_t		le32_t;
+typedef uint32_t		be32_t;
+typedef uint64_t		le64_t;
+typedef uint64_t		be64_t;
+
+#define true			1
+#define false			0
 
 #define NULL			((void *)0)
 
 #define __packed		__attribute__((packed))
 #define __noreturn		__attribute__((noreturn))
 #define __aligned(x)		__attribute__((aligned(x)))
+#define __always_inline	inline __attribute__((always_inline))
 
 #define likely(x) __builtin_expect((x), 1)
 #define unlikely(x) __builtin_expect((x), 0)

include/sbi/sbi_version.h

@@ -11,7 +11,7 @@
 #define __SBI_VERSION_H__
 
 #define OPENSBI_VERSION_MAJOR 1
-#define OPENSBI_VERSION_MINOR 2
+#define OPENSBI_VERSION_MINOR 4
 
 /**
  * OpenSBI 32-bit version with:

include/sbi_utils/fdt/fdt_fixup.h

@@ -9,6 +9,29 @@
 #ifndef __FDT_FIXUP_H__
 #define __FDT_FIXUP_H__
 
+struct sbi_cpu_idle_state {
+	const char *name;
+	uint32_t suspend_param;
+	bool local_timer_stop;
+	uint32_t entry_latency_us;
+	uint32_t exit_latency_us;
+	uint32_t min_residency_us;
+	uint32_t wakeup_latency_us;
+};
+
+/**
+ * Add CPU idle states to cpu nodes in the DT
+ *
+ * Add information about CPU idle states to the devicetree. This function
+ * assumes that CPU idle states are not already present in the devicetree, and
+ * that all CPU states are equally applicable to all CPUs.
+ *
+ * @param fdt: device tree blob
+ * @param states: array of idle state descriptions, ending with empty element
+ * @return zero on success and -ve on failure
+ */
+int fdt_add_cpu_idle_states(void *dtb, const struct sbi_cpu_idle_state *state);
+
 /**
  * Fix up the CPU node in the device tree
  *
@@ -70,20 +93,6 @@ void fdt_plic_fixup(void *fdt);
  */
 int fdt_reserved_memory_fixup(void *fdt);
 
-/**
- * Fix up the reserved memory subnodes in the device tree
- *
- * This routine adds the no-map property to the reserved memory subnodes so
- * that the OS does not map those PMP protected memory regions.
- *
- * Platform codes must call this helper in their final_init() after fdt_fixups()
- * if the OS should not map the PMP protected reserved regions.
- *
- * @param fdt: device tree blob
- * @return zero on success and -ve on failure
- */
-int fdt_reserved_memory_nomap_fixup(void *fdt);
-
 /**
  * General device tree fix-up
  *

include/sbi_utils/fdt/fdt_helper.h

@@ -11,7 +11,7 @@
 #define __FDT_HELPER_H__
 
 #include <sbi/sbi_types.h>
-#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_domain.h>
 
 struct fdt_match {
 	const char *compatible;
@@ -48,6 +48,9 @@ int fdt_parse_phandle_with_args(void *fdt, int nodeoff,
 int fdt_get_node_addr_size(void *fdt, int node, int index,
 			   uint64_t *addr, uint64_t *size);
 
+int fdt_get_node_addr_size_by_name(void *fdt, int node, const char *name,
+				   uint64_t *addr, uint64_t *size);
+
 bool fdt_node_is_enabled(void *fdt, int nodeoff);
 
 int fdt_parse_hart_id(void *fdt, int cpu_offset, u32 *hartid);
@@ -56,6 +59,9 @@ int fdt_parse_max_enabled_hart_id(void *fdt, u32 *max_hartid);
 
 int fdt_parse_timebase_frequency(void *fdt, unsigned long *freq);
 
+int fdt_parse_isa_extensions(void *fdt, unsigned int hard_id,
+			     unsigned long *extensions);
+
 int fdt_parse_gaisler_uart_node(void *fdt, int nodeoffset,
 				struct platform_uart_data *uart);
 
@@ -93,7 +99,8 @@ int fdt_parse_plic_node(void *fdt, int nodeoffset, struct plic_data *plic);
 
 int fdt_parse_plic(void *fdt, struct plic_data *plic, const char *compat);
 
-int fdt_parse_aclint_node(void *fdt, int nodeoffset, bool for_timer,
+int fdt_parse_aclint_node(void *fdt, int nodeoffset,
+			  bool for_timer, bool allow_regname,
 			  unsigned long *out_addr1, unsigned long *out_size1,
 			  unsigned long *out_addr2, unsigned long *out_size2,
 			  u32 *out_first_hartid, u32 *out_hart_count);
@@ -109,7 +116,7 @@ int fdt_parse_compat_addr(void *fdt, uint64_t *addr,
 
 static inline void *fdt_get_address(void)
 {
-	return sbi_scratch_thishart_arg1_ptr();
+	return (void *)root.next_arg1;
 }
 
 #endif /* __FDT_HELPER_H__ */

include/sbi_utils/fdt/fdt_pmu.h

@@ -13,6 +13,23 @@
 
 #include <sbi/sbi_types.h>
 
+struct fdt_pmu_hw_event_select_map {
+	uint32_t eidx;
+	uint64_t select;
+};
+
+struct fdt_pmu_hw_event_counter_map {
+	uint32_t eidx_start;
+	uint32_t eidx_end;
+	uint32_t ctr_map;
+};
+
+struct fdt_pmu_raw_event_counter_map {
+	uint64_t select;
+	uint64_t select_mask;
+	uint32_t ctr_map;
+};
+
 #ifdef CONFIG_FDT_PMU
 
 /**
@@ -26,7 +43,7 @@
  *
  * @param fdt device tree blob
  */
-void fdt_pmu_fixup(void *fdt);
+int fdt_pmu_fixup(void *fdt);
 
 /**
  * Setup PMU data from device tree
@@ -45,6 +62,11 @@ int fdt_pmu_setup(void *fdt);
  */
 uint64_t fdt_pmu_get_select_value(uint32_t event_idx);
 
+/** The event index to selector value table instance */
+extern struct fdt_pmu_hw_event_select_map fdt_pmu_evt_select[];
+/** The number of valid entries in fdt_pmu_evt_select[] */
+extern uint32_t hw_event_count;
+
 #else
 
 static inline void fdt_pmu_fixup(void *fdt) { }

include/sbi_utils/i2c/dw_i2c.h

@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2022 StarFive Technology Co., Ltd.
+ *
+ * Author: Minda Chen <minda.chen@starfivetech.com>
+ */
+
+#ifndef __DW_I2C_H__
+#define __DW_I2C_H__
+
+#include <sbi_utils/i2c/i2c.h>
+
+int dw_i2c_init(struct i2c_adapter *, int nodeoff);
+
+struct dw_i2c_adapter {
+	unsigned long addr;
+	struct i2c_adapter adapter;
+};
+
+#endif

include/sbi_utils/i2c/i2c.h

@@ -15,9 +15,6 @@
 
 /** Representation of a I2C adapter */
 struct i2c_adapter {
-	/** Pointer to I2C driver owning this I2C adapter */
-	void *driver;
-
 	/** Unique ID of the I2C adapter assigned by the driver */
 	int id;
 

include/sbi_utils/ipi/andes_plicsw.h

@@ -13,30 +13,23 @@
 #ifndef _IPI_ANDES_PLICSW_H_
 #define _IPI_ANDES_PLICSW_H_
 
-#define PLICSW_PRIORITY_BASE 0x4
+#define PLICSW_PRIORITY_BASE	0x4
 
-#define PLICSW_PENDING_BASE 0x1000
-#define PLICSW_PENDING_STRIDE 0x8
+#define PLICSW_PENDING_BASE	0x1000
 
-#define PLICSW_ENABLE_BASE 0x2000
-#define PLICSW_ENABLE_STRIDE 0x80
+#define PLICSW_ENABLE_BASE	0x2000
+#define PLICSW_ENABLE_STRIDE	0x80
 
-#define PLICSW_CONTEXT_BASE 0x200000
-#define PLICSW_CONTEXT_STRIDE 0x1000
-#define PLICSW_CONTEXT_CLAIM 0x4
+#define PLICSW_CONTEXT_BASE	0x200000
+#define PLICSW_CONTEXT_STRIDE	0x1000
+#define PLICSW_CONTEXT_CLAIM	0x4
 
-#define PLICSW_HART_MASK 0x01010101
-
-#define PLICSW_HART_MAX_NR 8
-
-#define PLICSW_REGION_ALIGN 0x1000
+#define PLICSW_REGION_ALIGN	0x1000
 
 struct plicsw_data {
 	unsigned long addr;
 	unsigned long size;
 	uint32_t hart_count;
-	/* hart id to source id table */
-	uint32_t source_id[PLICSW_HART_MAX_NR];
 };
 
 int plicsw_warm_ipi_init(void);

include/sbi_utils/irqchip/plic.h

@@ -14,6 +14,7 @@
 
 struct plic_data {
 	unsigned long addr;
+	unsigned long size;
 	unsigned long num_src;
 };
 

include/sbi_utils/regmap/fdt_regmap.h

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __FDT_REGMAP_H__
+#define __FDT_REGMAP_H__
+
+#include <sbi_utils/regmap/regmap.h>
+
+struct fdt_phandle_args;
+
+/** FDT based regmap driver */
+struct fdt_regmap {
+	const struct fdt_match *match_table;
+	int (*init)(void *fdt, int nodeoff, u32 phandle,
+		    const struct fdt_match *match);
+};
+
+/** Get regmap instance based on phandle */
+int fdt_regmap_get_by_phandle(void *fdt, u32 phandle,
+			      struct regmap **out_rmap);
+
+/** Get regmap instance based on "regmap' property of the specified DT node */
+int fdt_regmap_get(void *fdt, int nodeoff, struct regmap **out_rmap);
+
+#endif

include/sbi_utils/regmap/regmap.h

@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __REGMAP_H__
+#define __REGMAP_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_list.h>
+
+/** Representation of a regmap instance */
+struct regmap {
+	/** Uniquie ID of the regmap instance assigned by the driver */
+	unsigned int id;
+
+	/** Configuration of regmap registers */
+	int reg_shift;
+	int reg_stride;
+	unsigned int reg_base;
+	unsigned int reg_max;
+
+	/** Read a regmap register */
+	int (*reg_read)(struct regmap *rmap, unsigned int reg,
+			unsigned int *val);
+
+	/** Write a regmap register */
+	int (*reg_write)(struct regmap *rmap, unsigned int reg,
+			 unsigned int val);
+
+	/** Read-modify-write a regmap register */
+	int (*reg_update_bits)(struct regmap *rmap, unsigned int reg,
+			       unsigned int mask, unsigned int val);
+
+	/** List */
+	struct sbi_dlist node;
+};
+
+static inline struct regmap *to_regmap(struct sbi_dlist *node)
+{
+	return container_of(node, struct regmap, node);
+}
+
+/** Find a registered regmap instance */
+struct regmap *regmap_find(unsigned int id);
+
+/** Register a regmap instance */
+int regmap_add(struct regmap *rmap);
+
+/** Un-register a regmap instance */
+void regmap_remove(struct regmap *rmap);
+
+/** Read a register in a regmap instance */
+int regmap_read(struct regmap *rmap, unsigned int reg, unsigned int *val);
+
+/** Write a register in a regmap instance */
+int regmap_write(struct regmap *rmap, unsigned int reg, unsigned int val);
+
+/** Read-modify-write a register in a regmap instance */
+int regmap_update_bits(struct regmap *rmap, unsigned int reg,
+		       unsigned int mask, unsigned int val);
+
+#endif

include/sbi_utils/sys/atcsmu.h

@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2023 Andes Technology Corporation
+ */
+
+#ifndef _SYS_ATCSMU_H
+#define _SYS_ATCSMU_H
+
+#include <sbi/sbi_types.h>
+
+/* clang-format off */
+
+#define PCS0_WE_OFFSET		0x90
+#define PCSm_WE_OFFSET(i)	((i + 3) * 0x20 + PCS0_WE_OFFSET)
+
+#define PCS0_CTL_OFFSET		0x94
+#define PCSm_CTL_OFFSET(i)	((i + 3) * 0x20 + PCS0_CTL_OFFSET)
+#define PCS_CTL_CMD_SHIFT	0
+#define PCS_CTL_PARAM_SHIFT	3
+#define SLEEP_CMD		0x3
+#define WAKEUP_CMD		(0x0 | (1 << PCS_CTL_PARAM_SHIFT))
+#define LIGHTSLEEP_MODE		0
+#define DEEPSLEEP_MODE		1
+#define LIGHT_SLEEP_CMD		(SLEEP_CMD | (LIGHTSLEEP_MODE << PCS_CTL_PARAM_SHIFT))
+#define DEEP_SLEEP_CMD		(SLEEP_CMD | (DEEPSLEEP_MODE << PCS_CTL_PARAM_SHIFT))
+
+#define PCS0_CFG_OFFSET			0x80
+#define PCSm_CFG_OFFSET(i)		((i + 3) * 0x20 + PCS0_CFG_OFFSET)
+#define PCS_CFG_LIGHT_SLEEP_SHIFT	2
+#define PCS_CFG_LIGHT_SLEEP		(1 << PCS_CFG_LIGHT_SLEEP_SHIFT)
+#define PCS_CFG_DEEP_SLEEP_SHIFT	3
+#define PCS_CFG_DEEP_SLEEP		(1 << PCS_CFG_DEEP_SLEEP_SHIFT)
+
+#define RESET_VEC_LO_OFFSET	0x50
+#define RESET_VEC_HI_OFFSET	0x60
+#define RESET_VEC_8CORE_OFFSET	0x1a0
+#define HARTn_RESET_VEC_LO(n)	(RESET_VEC_LO_OFFSET + \
+			        ((n) < 4 ? 0 : RESET_VEC_8CORE_OFFSET) + \
+			        ((n) * 0x4))
+#define HARTn_RESET_VEC_HI(n)	(RESET_VEC_HI_OFFSET + \
+			        ((n) < 4 ? 0 : RESET_VEC_8CORE_OFFSET) + \
+			        ((n) * 0x4))
+
+#define PCS_MAX_NR  8
+#define FLASH_BASE  0x80000000ULL
+
+/* clang-format on */
+
+struct smu_data {
+	unsigned long addr;
+};
+
+int smu_set_wakeup_events(struct smu_data *smu, u32 events, u32 hartid);
+bool smu_support_sleep_mode(struct smu_data *smu, u32 sleep_mode, u32 hartid);
+int smu_set_command(struct smu_data *smu, u32 pcs_ctl, u32 hartid);
+int smu_set_reset_vector(struct smu_data *smu, ulong wakeup_addr, u32 hartid);
+
+#endif /* _SYS_ATCSMU_H */

include/sbi_utils/sys/sifive_test.h

@@ -1,17 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-3-Clause
- *
- * Copyright (c) 2020 Western Digital Corporation or its affiliates.
- *
- * Authors:
- *   Anup Patel <anup.patel@wdc.com>
- */
-
-#ifndef __SYS_SIFIVE_TEST_H__
-#define __SYS_SIFIVE_TEST_H__
-
-#include <sbi/sbi_types.h>
-
-int sifive_test_init(unsigned long base);
-
-#endif

lib/sbi/Kconfig

@@ -22,10 +22,22 @@ config SBI_ECALL_SRST
 	bool "System Reset extension"
 	default y
 
+config SBI_ECALL_SUSP
+	bool "System Suspend extension"
+	default y
+
 config SBI_ECALL_PMU
 	bool "Performance Monitoring Unit extension"
 	default y
 
+config SBI_ECALL_DBCN
+	bool "Debug Console extension"
+	default y
+
+config SBI_ECALL_CPPC
+	bool "CPPC extension"
+	default y
+
 config SBI_ECALL_LEGACY
 	bool "SBI v0.1 legacy extensions"
 	default y

lib/sbi/objects.mk

@@ -34,9 +34,18 @@ libsbi-objs-$(CONFIG_SBI_ECALL_HSM) += sbi_ecall_hsm.o
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_SRST) += ecall_srst
 libsbi-objs-$(CONFIG_SBI_ECALL_SRST) += sbi_ecall_srst.o
 
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_SUSP) += ecall_susp
+libsbi-objs-$(CONFIG_SBI_ECALL_SUSP) += sbi_ecall_susp.o
+
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_PMU) += ecall_pmu
 libsbi-objs-$(CONFIG_SBI_ECALL_PMU) += sbi_ecall_pmu.o
 
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_DBCN) += ecall_dbcn
+libsbi-objs-$(CONFIG_SBI_ECALL_DBCN) += sbi_ecall_dbcn.o
+
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_CPPC) += ecall_cppc
+libsbi-objs-$(CONFIG_SBI_ECALL_CPPC) += sbi_ecall_cppc.o
+
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_LEGACY) += ecall_legacy
 libsbi-objs-$(CONFIG_SBI_ECALL_LEGACY) += sbi_ecall_legacy.o
 
@@ -50,6 +59,7 @@ libsbi-objs-y += sbi_domain.o
 libsbi-objs-y += sbi_emulate_csr.o
 libsbi-objs-y += sbi_fifo.o
 libsbi-objs-y += sbi_hart.o
+libsbi-objs-y += sbi_heap.o
 libsbi-objs-y += sbi_math.o
 libsbi-objs-y += sbi_hfence.o
 libsbi-objs-y += sbi_hsm.o
@@ -68,3 +78,4 @@ libsbi-objs-y += sbi_tlb.o
 libsbi-objs-y += sbi_trap.o
 libsbi-objs-y += sbi_unpriv.o
 libsbi-objs-y += sbi_expected_trap.o
+libsbi-objs-y += sbi_cppc.o

lib/sbi/riscv_asm.c

@@ -128,6 +128,8 @@ unsigned long csr_read_num(int csr_num)
 	switchcase_csr_read_8(CSR_MHPMCOUNTER8, ret)
 	switchcase_csr_read_16(CSR_MHPMCOUNTER16, ret)
 	switchcase_csr_read(CSR_MCOUNTINHIBIT, ret)
+	switchcase_csr_read(CSR_MCYCLECFG, ret)
+	switchcase_csr_read(CSR_MINSTRETCFG, ret)
 	switchcase_csr_read(CSR_MHPMEVENT3, ret)
 	switchcase_csr_read_4(CSR_MHPMEVENT4, ret)
 	switchcase_csr_read_8(CSR_MHPMEVENT8, ret)
@@ -139,6 +141,12 @@ unsigned long csr_read_num(int csr_num)
 	switchcase_csr_read_4(CSR_MHPMCOUNTER4H, ret)
 	switchcase_csr_read_8(CSR_MHPMCOUNTER8H, ret)
 	switchcase_csr_read_16(CSR_MHPMCOUNTER16H, ret)
+	/**
+	 * The CSR range M[CYCLE, INSTRET]CFGH are available only if smcntrpmf
+	 * extension is present. The caller must ensure that.
+	 */
+	switchcase_csr_read(CSR_MCYCLECFGH, ret)
+	switchcase_csr_read(CSR_MINSTRETCFGH, ret)
 	/**
 	 * The CSR range MHPMEVENT[3-16]H are available only if sscofpmf
 	 * extension is present. The caller must ensure that.
@@ -152,7 +160,7 @@ unsigned long csr_read_num(int csr_num)
 	default:
 		sbi_panic("%s: Unknown CSR %#x", __func__, csr_num);
 		break;
-	};
+	}
 
 	return ret;
 
@@ -206,12 +214,16 @@ void csr_write_num(int csr_num, unsigned long val)
 	switchcase_csr_write_4(CSR_MHPMCOUNTER4H, val)
 	switchcase_csr_write_8(CSR_MHPMCOUNTER8H, val)
 	switchcase_csr_write_16(CSR_MHPMCOUNTER16H, val)
+	switchcase_csr_write(CSR_MCYCLECFGH, val)
+	switchcase_csr_write(CSR_MINSTRETCFGH, val)
 	switchcase_csr_write(CSR_MHPMEVENT3H, val)
 	switchcase_csr_write_4(CSR_MHPMEVENT4H, val)
 	switchcase_csr_write_8(CSR_MHPMEVENT8H, val)
 	switchcase_csr_write_16(CSR_MHPMEVENT16H, val)
 #endif
 	switchcase_csr_write(CSR_MCOUNTINHIBIT, val)
+	switchcase_csr_write(CSR_MCYCLECFG, val)
+	switchcase_csr_write(CSR_MINSTRETCFG, val)
 	switchcase_csr_write(CSR_MHPMEVENT3, val)
 	switchcase_csr_write_4(CSR_MHPMEVENT4, val)
 	switchcase_csr_write_8(CSR_MHPMEVENT8, val)
@@ -220,7 +232,7 @@ void csr_write_num(int csr_num, unsigned long val)
 	default:
 		sbi_panic("%s: Unknown CSR %#x", __func__, csr_num);
 		break;
-	};
+	}
 
 #undef switchcase_csr_write_64
 #undef switchcase_csr_write_32
@@ -246,6 +258,48 @@ static unsigned long ctz(unsigned long x)
 	return ret;
 }
 
+int pmp_disable(unsigned int n)
+{
+	int pmpcfg_csr, pmpcfg_shift;
+	unsigned long cfgmask, pmpcfg;
+
+	if (n >= PMP_COUNT)
+		return SBI_EINVAL;
+
+#if __riscv_xlen == 32
+	pmpcfg_csr   = CSR_PMPCFG0 + (n >> 2);
+	pmpcfg_shift = (n & 3) << 3;
+#elif __riscv_xlen == 64
+	pmpcfg_csr   = (CSR_PMPCFG0 + (n >> 2)) & ~1;
+	pmpcfg_shift = (n & 7) << 3;
+#else
+# error "Unexpected __riscv_xlen"
+#endif
+
+	/* Clear the address matching bits to disable the pmp entry */
+	cfgmask = ~(0xffUL << pmpcfg_shift);
+	pmpcfg	= (csr_read_num(pmpcfg_csr) & cfgmask);
+
+	csr_write_num(pmpcfg_csr, pmpcfg);
+
+	return SBI_OK;
+}
+
+int is_pmp_entry_mapped(unsigned long entry)
+{
+	unsigned long prot;
+	unsigned long addr;
+	unsigned long log2len;
+
+	pmp_get(entry, &prot, &addr, &log2len);
+
+	/* If address matching bits are non-zero, the entry is enable */
+	if (prot & PMP_A)
+		return true;
+
+	return false;
+}
+
 int pmp_set(unsigned int n, unsigned long prot, unsigned long addr,
 	    unsigned long log2len)
 {

lib/sbi/riscv_atomic.c

@@ -12,6 +12,10 @@
 #include <sbi/riscv_atomic.h>
 #include <sbi/riscv_barrier.h>
 
+#ifndef __riscv_atomic
+#error "opensbi strongly relies on the A extension of RISC-V"
+#endif
+
 long atomic_read(atomic_t *atom)
 {
 	long ret = atom->counter;
@@ -79,175 +83,51 @@ long atomic_sub_return(atomic_t *atom, long value)
 		(__typeof__(*(ptr))) __axchg((ptr), _x_, sizeof(*(ptr)));	\
 	})
 
-
-#define __xchg(ptr, new, size)                                            \
-	({                                                                \
-		__typeof__(ptr) __ptr	 = (ptr);                         \
-		__typeof__(*(ptr)) __new = (new);                         \
-		__typeof__(*(ptr)) __ret;                                 \
-		register unsigned int __rc;                               \
-		switch (size) {                                           \
-		case 4:                                                   \
-			__asm__ __volatile__("0:	lr.w %0, %2\n"    \
-					     "	sc.w.rl %1, %z3, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__new)                \
-					     : "memory");                 \
-			break;                                            \
-		case 8:                                                   \
-			__asm__ __volatile__("0:	lr.d %0, %2\n"    \
-					     "	sc.d.rl %1, %z3, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__new)                \
-					     : "memory");                 \
-			break;                                            \
-		default:                                                  \
-			break;                                            \
-		}                                                         \
-		__ret;                                                    \
-	})
-
-#define xchg(ptr, n)                                                     \
-	({                                                               \
-		__typeof__(*(ptr)) _n_ = (n);                            \
-		(__typeof__(*(ptr))) __xchg((ptr), _n_, sizeof(*(ptr))); \
-	})
-
-#define __cmpxchg(ptr, old, new, size)                                    \
-	({                                                                \
-		__typeof__(ptr) __ptr	 = (ptr);                         \
-		__typeof__(*(ptr)) __old = (old);                         \
-		__typeof__(*(ptr)) __new = (new);                         \
-		__typeof__(*(ptr)) __ret;                                 \
-		register unsigned int __rc;                               \
-		switch (size) {                                           \
-		case 4:                                                   \
-			__asm__ __volatile__("0:	lr.w %0, %2\n"    \
-					     "	bne  %0, %z3, 1f\n"       \
-					     "	sc.w.rl %1, %z4, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     "1:\n"                       \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__old), "rJ"(__new)   \
-					     : "memory");                 \
-			break;                                            \
-		case 8:                                                   \
-			__asm__ __volatile__("0:	lr.d %0, %2\n"    \
-					     "	bne %0, %z3, 1f\n"        \
-					     "	sc.d.rl %1, %z4, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     "1:\n"                       \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__old), "rJ"(__new)   \
-					     : "memory");                 \
-			break;                                            \
-		default:                                                  \
-			break;                                            \
-		}                                                         \
-		__ret;                                                    \
-	})
-
-#define cmpxchg(ptr, o, n)                                          \
-	({                                                          \
-		__typeof__(*(ptr)) _o_ = (o);                       \
-		__typeof__(*(ptr)) _n_ = (n);                       \
-		(__typeof__(*(ptr)))                                \
-			__cmpxchg((ptr), _o_, _n_, sizeof(*(ptr))); \
-	})
-
 long atomic_cmpxchg(atomic_t *atom, long oldval, long newval)
 {
-#ifdef __riscv_atomic
 	return __sync_val_compare_and_swap(&atom->counter, oldval, newval);
-#else
-	return cmpxchg(&atom->counter, oldval, newval);
-#endif
 }
 
 long atomic_xchg(atomic_t *atom, long newval)
 {
 	/* Atomically set new value and return old value. */
-#ifdef __riscv_atomic
 	return axchg(&atom->counter, newval);
-#else
-	return xchg(&atom->counter, newval);
-#endif
 }
 
 unsigned int atomic_raw_xchg_uint(volatile unsigned int *ptr,
 				  unsigned int newval)
 {
 	/* Atomically set new value and return old value. */
-#ifdef __riscv_atomic
 	return axchg(ptr, newval);
-#else
-	return xchg(ptr, newval);
-#endif
 }
 
 unsigned long atomic_raw_xchg_ulong(volatile unsigned long *ptr,
 				    unsigned long newval)
 {
 	/* Atomically set new value and return old value. */
-#ifdef __riscv_atomic
 	return axchg(ptr, newval);
-#else
-	return xchg(ptr, newval);
-#endif
 }
 
-#if (__SIZEOF_POINTER__ == 8)
-#define __AMO(op) "amo" #op ".d"
-#elif (__SIZEOF_POINTER__ == 4)
-#define __AMO(op) "amo" #op ".w"
-#else
-#error "Unexpected __SIZEOF_POINTER__"
-#endif
-
-#define __atomic_op_bit_ord(op, mod, nr, addr, ord)                          \
-	({                                                                   \
-		unsigned long __res, __mask;                                 \
-		__mask = BIT_MASK(nr);                                       \
-		__asm__ __volatile__(__AMO(op) #ord " %0, %2, %1"            \
-				     : "=r"(__res), "+A"(addr[BIT_WORD(nr)]) \
-				     : "r"(mod(__mask))                      \
-				     : "memory");                            \
-		__res;                                                       \
-	})
-
-#define __atomic_op_bit(op, mod, nr, addr) \
-	__atomic_op_bit_ord(op, mod, nr, addr, .aqrl)
-
-/* Bitmask modifiers */
-#define __NOP(x) (x)
-#define __NOT(x) (~(x))
-
-inline int atomic_raw_set_bit(int nr, volatile unsigned long *addr)
+int atomic_raw_set_bit(int nr, volatile unsigned long *addr)
 {
-	return __atomic_op_bit(or, __NOP, nr, addr);
+	unsigned long res, mask = BIT_MASK(nr);
+	res = __atomic_fetch_or(&addr[BIT_WORD(nr)], mask, __ATOMIC_RELAXED);
+	return res & mask ? 1 : 0;
 }
 
-inline int atomic_raw_clear_bit(int nr, volatile unsigned long *addr)
+int atomic_raw_clear_bit(int nr, volatile unsigned long *addr)
 {
-	return __atomic_op_bit(and, __NOT, nr, addr);
+	unsigned long res, mask = BIT_MASK(nr);
+	res = __atomic_fetch_and(&addr[BIT_WORD(nr)], ~mask, __ATOMIC_RELAXED);
+	return res & mask ? 1 : 0;
 }
 
-inline int atomic_set_bit(int nr, atomic_t *atom)
+int atomic_set_bit(int nr, atomic_t *atom)
 {
 	return atomic_raw_set_bit(nr, (unsigned long *)&atom->counter);
 }
 
-inline int atomic_clear_bit(int nr, atomic_t *atom)
+int atomic_clear_bit(int nr, atomic_t *atom)
 {
 	return atomic_raw_clear_bit(nr, (unsigned long *)&atom->counter);
 }

lib/sbi/sbi_console.c

@@ -12,17 +12,22 @@
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+
+#define CONSOLE_TBUF_MAX 256
 
 static const struct sbi_console_device *console_dev = NULL;
+static char console_tbuf[CONSOLE_TBUF_MAX];
+static u32 console_tbuf_len;
 static spinlock_t console_out_lock	       = SPIN_LOCK_INITIALIZER;
 
 bool sbi_isprintable(char c)
 {
 	if (((31 < c) && (c < 127)) || (c == '\f') || (c == '\r') ||
 	    (c == '\n') || (c == '\t')) {
-		return TRUE;
+		return true;
 	}
-	return FALSE;
+	return false;
 }
 
 int sbi_getc(void)
@@ -32,25 +37,57 @@ int sbi_getc(void)
 	return -1;
 }
 
+static unsigned long nputs(const char *str, unsigned long len)
+{
+	unsigned long i;
+
+	if (console_dev) {
+		if (console_dev->console_puts)
+			return console_dev->console_puts(str, len);
+		else if (console_dev->console_putc) {
+			for (i = 0; i < len; i++) {
+				if (str[i] == '\n')
+					console_dev->console_putc('\r');
+				console_dev->console_putc(str[i]);
+			}
+		}
+	}
+	return len;
+}
+
+static void nputs_all(const char *str, unsigned long len)
+{
+	unsigned long p = 0;
+
+	while (p < len)
+		p += nputs(&str[p], len - p);
+}
+
 void sbi_putc(char ch)
 {
-	if (console_dev && console_dev->console_putc) {
-		if (ch == '\n')
-			console_dev->console_putc('\r');
-		console_dev->console_putc(ch);
-	}
+	nputs_all(&ch, 1);
 }
 
 void sbi_puts(const char *str)
 {
+	unsigned long len = sbi_strlen(str);
+
 	spin_lock(&console_out_lock);
-	while (*str) {
-		sbi_putc(*str);
-		str++;
-	}
+	nputs_all(str, len);
 	spin_unlock(&console_out_lock);
 }
 
+unsigned long sbi_nputs(const char *str, unsigned long len)
+{
+	unsigned long ret;
+
+	spin_lock(&console_out_lock);
+	ret = nputs(str, len);
+	spin_unlock(&console_out_lock);
+
+	return ret;
+}
+
 void sbi_gets(char *s, int maxwidth, char endchar)
 {
 	int ch;
@@ -64,9 +101,26 @@ void sbi_gets(char *s, int maxwidth, char endchar)
 	*retval = '\0';
 }
 
+unsigned long sbi_ngets(char *str, unsigned long len)
+{
+	int ch;
+	unsigned long i;
+
+	for (i = 0; i < len; i++) {
+		ch = sbi_getc();
+		if (ch < 0)
+			break;
+		str[i] = ch;
+	}
+
+	return i;
+}
+
 #define PAD_RIGHT 1
 #define PAD_ZERO 2
 #define PAD_ALTERNATE 4
+#define PAD_SIGN 8
+#define USE_TBUF 16
 #define PRINT_BUF_LEN 64
 
 #define va_start(v, l) __builtin_va_start((v), l)
@@ -74,7 +128,7 @@ void sbi_gets(char *s, int maxwidth, char endchar)
 #define va_arg __builtin_va_arg
 typedef __builtin_va_list va_list;
 
-static void printc(char **out, u32 *out_len, char ch)
+static void printc(char **out, u32 *out_len, char ch, int flags)
 {
 	if (!out) {
 		sbi_putc(ch);
@@ -88,60 +142,66 @@ static void printc(char **out, u32 *out_len, char ch)
 	if (!out_len || *out_len > 1) {
 		*(*out)++ = ch;
 		**out = '\0';
+		if (out_len) {
+			--(*out_len);
+			if ((flags & USE_TBUF) && *out_len == 1) {
+				nputs_all(console_tbuf, CONSOLE_TBUF_MAX - *out_len);
+				*out = console_tbuf;
+				*out_len = CONSOLE_TBUF_MAX;
+			}
+		}
 	}
-
-	if (out_len && *out_len > 0)
-		--(*out_len);
 }
 
 static int prints(char **out, u32 *out_len, const char *string, int width,
 		  int flags)
 {
-	int pc	     = 0;
-	char padchar = ' ';
-
-	if (width > 0) {
-		int len = 0;
-		const char *ptr;
-		for (ptr = string; *ptr; ++ptr)
-			++len;
-		if (len >= width)
-			width = 0;
-		else
-			width -= len;
-		if (flags & PAD_ZERO)
-			padchar = '0';
-	}
+	int pc = 0;
+	width -= sbi_strlen(string);
 	if (!(flags & PAD_RIGHT)) {
 		for (; width > 0; --width) {
-			printc(out, out_len, padchar);
+			printc(out, out_len, flags & PAD_ZERO ? '0' : ' ', flags);
 			++pc;
 		}
 	}
 	for (; *string; ++string) {
-		printc(out, out_len, *string);
+		printc(out, out_len, *string, flags);
 		++pc;
 	}
 	for (; width > 0; --width) {
-		printc(out, out_len, padchar);
+		printc(out, out_len, ' ', flags);
 		++pc;
 	}
 
 	return pc;
 }
 
-static int printi(char **out, u32 *out_len, long long i, int b, int sg,
-		  int width, int flags, int letbase)
+static int printi(char **out, u32 *out_len, long long i,
+		  int width, int flags, int type)
 {
-	char print_buf[PRINT_BUF_LEN];
-	char *s;
-	int neg = 0, pc = 0;
-	u64 t;
-	unsigned long long u = i;
+	int pc = 0;
+	char *s, sign = 0, letbase, print_buf[PRINT_BUF_LEN];
+	unsigned long long u, b, t;
 
-	if (sg && b == 10 && i < 0) {
-		neg = 1;
-		u   = -i;
+	b = 10;
+	letbase = 'a';
+	if (type == 'o')
+		b = 8;
+	else if (type == 'x' || type == 'X' || type == 'p' || type == 'P') {
+		b = 16;
+		letbase &= ~0x20;
+		letbase |= type & 0x20;
+	}
+
+	u = i;
+	sign = 0;
+	if (type == 'i' || type == 'd') {
+		if ((flags & PAD_SIGN) && i > 0)
+			sign = '+';
+		if (i < 0) {
+			sign = '-';
+			u = -i;
+		}
 	}
 
 	s  = print_buf + PRINT_BUF_LEN - 1;
@@ -159,23 +219,33 @@ static int printi(char **out, u32 *out_len, long long i, int b, int sg,
 		}
 	}
 
-	if (flags & PAD_ALTERNATE) {
-		if ((b == 16) && (letbase == 'A')) {
-			*--s = 'X';
-		} else if ((b == 16) && (letbase == 'a')) {
-			*--s = 'x';
-		}
-		*--s = '0';
-	}
-
-	if (neg) {
-		if (width && (flags & PAD_ZERO)) {
-			printc(out, out_len, '-');
+	if (flags & PAD_ZERO) {
+		if (sign) {
+			printc(out, out_len, sign, flags);
 			++pc;
 			--width;
-		} else {
-			*--s = '-';
 		}
+		if (i && (flags & PAD_ALTERNATE)) {
+			if (b == 16 || b == 8) {
+				printc(out, out_len, '0', flags);
+				++pc;
+				--width;
+			}
+			if (b == 16) {
+				printc(out, out_len, 'x' - 'a' + letbase, flags);
+				++pc;
+				--width;
+			}
+		}
+	} else {
+		if (i && (flags & PAD_ALTERNATE)) {
+			if (b == 16)
+				*--s = 'x' - 'a' + letbase;
+			if (b == 16 || b == 8)
+				*--s = '0';
+		}
+		if (sign)
+			*--s = sign;
 	}
 
 	return pc + prints(out, out_len, s, width, flags);
@@ -183,32 +253,68 @@ static int printi(char **out, u32 *out_len, long long i, int b, int sg,
 
 static int print(char **out, u32 *out_len, const char *format, va_list args)
 {
-	int width, flags;
-	int pc = 0;
-	char scr[2];
-	unsigned long long tmp;
+	bool flags_done;
+	int width, flags, pc = 0;
+	char type, scr[2], *tout;
+	bool use_tbuf = (!out) ? true : false;
+
+	/*
+	 * The console_tbuf is protected by console_out_lock and
+	 * print() is always called with console_out_lock held
+	 * when out == NULL.
+	 */
+	if (use_tbuf) {
+		console_tbuf_len = CONSOLE_TBUF_MAX;
+		tout = console_tbuf;
+		out = &tout;
+		out_len = &console_tbuf_len;
+	}
+
+	/* handle special case: *out_len == 1*/
+	if (out) {
+		if(!out_len || *out_len)
+			**out = '\0';
+	}
 
 	for (; *format != 0; ++format) {
+		width = flags = 0;
+		if (use_tbuf)
+			flags |= USE_TBUF;
 		if (*format == '%') {
 			++format;
-			width = flags = 0;
 			if (*format == '\0')
 				break;
 			if (*format == '%')
 				goto literal;
 			/* Get flags */
-			if (*format == '-') {
-				++format;
-				flags = PAD_RIGHT;
-			}
-			if (*format == '#') {
-				++format;
-				flags |= PAD_ALTERNATE;
-			}
-			while (*format == '0') {
-				++format;
-				flags |= PAD_ZERO;
+			flags_done = false;
+			while (!flags_done) {
+				switch (*format) {
+				case '-':
+					flags |= PAD_RIGHT;
+					break;
+				case '+':
+					flags |= PAD_SIGN;
+					break;
+				case '#':
+					flags |= PAD_ALTERNATE;
+					break;
+				case '0':
+					flags |= PAD_ZERO;
+					break;
+				case ' ':
+				case '\'':
+					/* Ignored flags, do nothing */
+					break;
+				default:
+					flags_done = true;
+					break;
+				}
+				if (!flags_done)
+					++format;
 			}
+			if (flags & PAD_RIGHT)
+				flags &= ~PAD_ZERO;
 			/* Get width */
 			for (; *format >= '0' && *format <= '9'; ++format) {
 				width *= 10;
@@ -222,83 +328,47 @@ static int print(char **out, u32 *out_len, const char *format, va_list args)
 			}
 			if ((*format == 'd') || (*format == 'i')) {
 				pc += printi(out, out_len, va_arg(args, int),
-					     10, 1, width, flags, '0');
+					     width, flags, *format);
 				continue;
 			}
-			if (*format == 'x') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned int), 16, 0,
-					     width, flags, 'a');
+			if ((*format == 'u') || (*format == 'o')
+					 || (*format == 'x') || (*format == 'X')) {
+				pc += printi(out, out_len, va_arg(args, unsigned int),
+					     width, flags, *format);
 				continue;
 			}
-			if (*format == 'X') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned int), 16, 0,
-					     width, flags, 'A');
+			if ((*format == 'p') || (*format == 'P')) {
+				pc += printi(out, out_len, (uintptr_t)va_arg(args, void*),
+					     width, flags, *format);
 				continue;
 			}
-			if (*format == 'u') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned int), 10, 0,
-					     width, flags, 'a');
-				continue;
-			}
-			if (*format == 'p') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned long), 16, 0,
-					     width, flags, 'a');
-				continue;
-			}
-			if (*format == 'P') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned long), 16, 0,
-					     width, flags, 'A');
-				continue;
-			}
-			if (*format == 'l' && *(format + 1) == 'l') {
-				tmp = va_arg(args, unsigned long long);
-				if (*(format + 2) == 'u') {
-					format += 2;
-					pc += printi(out, out_len, tmp, 10, 0,
-						     width, flags, 'a');
-				} else if (*(format + 2) == 'x') {
-					format += 2;
-					pc += printi(out, out_len, tmp, 16, 0,
-						     width, flags, 'a');
-				} else if (*(format + 2) == 'X') {
-					format += 2;
-					pc += printi(out, out_len, tmp, 16, 0,
-						     width, flags, 'A');
-				} else {
-					format += 1;
-					pc += printi(out, out_len, tmp, 10, 1,
-						     width, flags, '0');
+			if (*format == 'l') {
+				type = 'i';
+				if (format[1] == 'l') {
+					++format;
+					if ((format[1] == 'u') || (format[1] == 'o')
+							|| (format[1] == 'd') || (format[1] == 'i')
+							|| (format[1] == 'x') || (format[1] == 'X')) {
+						++format;
+						type = *format;
+					}
+					pc += printi(out, out_len, va_arg(args, long long),
+						width, flags, type);
+					continue;
 				}
-				continue;
-			} else if (*format == 'l') {
-				if (*(format + 1) == 'u') {
-					format += 1;
-					pc += printi(
-						out, out_len,
-						va_arg(args, unsigned long), 10,
-						0, width, flags, 'a');
-				} else if (*(format + 1) == 'x') {
-					format += 1;
-					pc += printi(
-						out, out_len,
-						va_arg(args, unsigned long), 16,
-						0, width, flags, 'a');
-				} else if (*(format + 1) == 'X') {
-					format += 1;
-					pc += printi(
-						out, out_len,
-						va_arg(args, unsigned long), 16,
-						0, width, flags, 'A');
-				} else {
-					pc += printi(out, out_len,
-						     va_arg(args, long), 10, 1,
-						     width, flags, '0');
+				if ((format[1] == 'u') || (format[1] == 'o')
+						|| (format[1] == 'd') || (format[1] == 'i')
+						|| (format[1] == 'x') || (format[1] == 'X')) {
+					++format;
+					type = *format;
 				}
+				if ((type == 'd') || (type == 'i'))
+					pc += printi(out, out_len, va_arg(args, long),
+					     width, flags, type);
+				else
+					pc += printi(out, out_len, va_arg(args, unsigned long),
+					     width, flags, type);
+				continue;
 			}
 			if (*format == 'c') {
 				/* char are converted to int then pushed on the stack */
@@ -309,11 +379,14 @@ static int print(char **out, u32 *out_len, const char *format, va_list args)
 			}
 		} else {
 literal:
-			printc(out, out_len, *format);
+			printc(out, out_len, *format, flags);
 			++pc;
 		}
 	}
 
+	if (use_tbuf && console_tbuf_len < CONSOLE_TBUF_MAX)
+		nputs_all(console_tbuf, CONSOLE_TBUF_MAX - console_tbuf_len);
+
 	return pc;
 }
 
@@ -407,5 +480,11 @@ void sbi_console_set_device(const struct sbi_console_device *dev)
 
 int sbi_console_init(struct sbi_scratch *scratch)
 {
-	return sbi_platform_console_init(sbi_platform_ptr(scratch));
+	int rc = sbi_platform_console_init(sbi_platform_ptr(scratch));
+
+	/* console is not a necessary device */
+	if (rc == SBI_ENODEV)
+		return 0;
+
+	return rc;
 }

lib/sbi/sbi_cppc.c

@@ -0,0 +1,110 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ */
+
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_cppc.h>
+
+static const struct sbi_cppc_device *cppc_dev = NULL;
+
+const struct sbi_cppc_device *sbi_cppc_get_device(void)
+{
+	return cppc_dev;
+}
+
+void sbi_cppc_set_device(const struct sbi_cppc_device *dev)
+{
+	if (!dev || cppc_dev)
+		return;
+
+	cppc_dev = dev;
+}
+
+static bool sbi_cppc_is_reserved(unsigned long reg)
+{
+	if ((reg > SBI_CPPC_ACPI_LAST && reg < SBI_CPPC_TRANSITION_LATENCY) ||
+	    reg > SBI_CPPC_NON_ACPI_LAST)
+		return true;
+
+	return false;
+}
+
+static bool sbi_cppc_readable(unsigned long reg)
+{
+	/* there are no write-only cppc registers currently */
+	return true;
+}
+
+static bool sbi_cppc_writable(unsigned long reg)
+{
+	switch (reg) {
+	case SBI_CPPC_HIGHEST_PERF:
+	case SBI_CPPC_NOMINAL_PERF:
+	case SBI_CPPC_LOW_NON_LINEAR_PERF:
+	case SBI_CPPC_LOWEST_PERF:
+	case SBI_CPPC_GUARANTEED_PERF:
+	case SBI_CPPC_CTR_WRAP_TIME:
+	case SBI_CPPC_REFERENCE_CTR:
+	case SBI_CPPC_DELIVERED_CTR:
+	case SBI_CPPC_REFERENCE_PERF:
+	case SBI_CPPC_LOWEST_FREQ:
+	case SBI_CPPC_NOMINAL_FREQ:
+	case SBI_CPPC_TRANSITION_LATENCY:
+		return false;
+	}
+
+	return true;
+}
+
+int sbi_cppc_probe(unsigned long reg)
+{
+	if (!cppc_dev || !cppc_dev->cppc_probe)
+		return SBI_EFAIL;
+
+	/* Check whether register is reserved */
+	if (sbi_cppc_is_reserved(reg))
+		return SBI_ERR_INVALID_PARAM;
+
+	return cppc_dev->cppc_probe(reg);
+}
+
+int sbi_cppc_read(unsigned long reg, uint64_t *val)
+{
+	int ret;
+
+	if (!cppc_dev || !cppc_dev->cppc_read)
+		return SBI_EFAIL;
+
+	/* Check whether register is implemented */
+	ret = sbi_cppc_probe(reg);
+	if (ret <= 0)
+		return SBI_ERR_NOT_SUPPORTED;
+
+	/* Check whether the register is write-only */
+	if (!sbi_cppc_readable(reg))
+		return SBI_ERR_DENIED;
+
+	return cppc_dev->cppc_read(reg, val);
+}
+
+int sbi_cppc_write(unsigned long reg, uint64_t val)
+{
+	int ret;
+
+	if (!cppc_dev || !cppc_dev->cppc_write)
+		return SBI_EFAIL;
+
+	/* Check whether register is implemented */
+	ret = sbi_cppc_probe(reg);
+	if (ret <= 0)
+		return SBI_ERR_NOT_SUPPORTED;
+
+	/* Check whether the register is read-only */
+	if (!sbi_cppc_writable(reg))
+		return SBI_ERR_DENIED;
+
+	return cppc_dev->cppc_write(reg, val);
+}

lib/sbi/sbi_domain.c

@@ -11,67 +11,77 @@
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_domain.h>
 #include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_heap.h>
 #include <sbi/sbi_hsm.h>
 #include <sbi/sbi_math.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
 
-struct sbi_domain *hartid_to_domain_table[SBI_HARTMASK_MAX_BITS] = { 0 };
-struct sbi_domain *domidx_to_domain_table[SBI_DOMAIN_MAX_INDEX] = { 0 };
+/*
+ * We allocate an extra element because sbi_domain_for_each() expects
+ * the array to be null-terminated.
+ */
+struct sbi_domain *domidx_to_domain_table[SBI_DOMAIN_MAX_INDEX + 1] = { 0 };
 static u32 domain_count = 0;
 static bool domain_finalized = false;
 
-static struct sbi_hartmask root_hmask = { 0 };
-
 #define ROOT_REGION_MAX	16
 static u32 root_memregs_count = 0;
-static struct sbi_domain_memregion root_fw_region;
-static struct sbi_domain_memregion root_memregs[ROOT_REGION_MAX + 1] = { 0 };
 
 struct sbi_domain root = {
 	.name = "root",
-	.possible_harts = &root_hmask,
-	.regions = root_memregs,
-	.system_reset_allowed = TRUE,
+	.possible_harts = NULL,
+	.regions = NULL,
+	.system_reset_allowed = true,
+	.system_suspend_allowed = true,
+	.fw_region_inited = false,
 };
 
+static unsigned long domain_hart_ptr_offset;
+
+struct sbi_domain *sbi_hartindex_to_domain(u32 hartindex)
+{
+	struct sbi_scratch *scratch;
+
+	scratch = sbi_hartindex_to_scratch(hartindex);
+	if (!scratch || !domain_hart_ptr_offset)
+		return NULL;
+
+	return sbi_scratch_read_type(scratch, void *, domain_hart_ptr_offset);
+}
+
+static void update_hartindex_to_domain(u32 hartindex, struct sbi_domain *dom)
+{
+	struct sbi_scratch *scratch;
+
+	scratch = sbi_hartindex_to_scratch(hartindex);
+	if (!scratch)
+		return;
+
+	sbi_scratch_write_type(scratch, void *, domain_hart_ptr_offset, dom);
+}
+
 bool sbi_domain_is_assigned_hart(const struct sbi_domain *dom, u32 hartid)
 {
 	if (dom)
-		return sbi_hartmask_test_hart(hartid, &dom->assigned_harts);
+		return sbi_hartmask_test_hartid(hartid, &dom->assigned_harts);
 
-	return FALSE;
+	return false;
 }
 
 ulong sbi_domain_get_assigned_hartmask(const struct sbi_domain *dom,
 				       ulong hbase)
 {
-	ulong ret, bword, boff;
-
-	if (!dom)
-		return 0;
-
-	bword = BIT_WORD(hbase);
-	boff = BIT_WORD_OFFSET(hbase);
-
-	ret = sbi_hartmask_bits(&dom->assigned_harts)[bword++] >> boff;
-	if (boff && bword < BIT_WORD(SBI_HARTMASK_MAX_BITS)) {
-		ret |= (sbi_hartmask_bits(&dom->assigned_harts)[bword] &
-			(BIT(boff) - 1UL)) << (BITS_PER_LONG - boff);
+	ulong ret = 0;
+	for (int i = 0; i < 8 * sizeof(ret); i++) {
+		if (sbi_domain_is_assigned_hart(dom, hbase + i))
+			ret |= 1UL << i;
 	}
 
 	return ret;
 }
 
-static void domain_memregion_initfw(struct sbi_domain_memregion *reg)
-{
-	if (!reg)
-		return;
-
-	sbi_memcpy(reg, &root_fw_region, sizeof(*reg));
-}
-
 void sbi_domain_memregion_init(unsigned long addr,
 				unsigned long size,
 				unsigned long flags,
@@ -105,54 +115,64 @@ bool sbi_domain_check_addr(const struct sbi_domain *dom,
 			   unsigned long addr, unsigned long mode,
 			   unsigned long access_flags)
 {
-	bool rmmio, mmio = FALSE;
+	bool rmmio, mmio = false;
 	struct sbi_domain_memregion *reg;
-	unsigned long rstart, rend, rflags, rwx = 0;
+	unsigned long rstart, rend, rflags, rwx = 0, rrwx = 0;
 
 	if (!dom)
-		return FALSE;
+		return false;
 
+	/*
+	 * Use M_{R/W/X} bits because the SU-bits are at the
+	 * same relative offsets. If the mode is not M, the SU
+	 * bits will fall at same offsets after the shift.
+	 */
 	if (access_flags & SBI_DOMAIN_READ)
-		rwx |= SBI_DOMAIN_MEMREGION_READABLE;
+		rwx |= SBI_DOMAIN_MEMREGION_M_READABLE;
+
 	if (access_flags & SBI_DOMAIN_WRITE)
-		rwx |= SBI_DOMAIN_MEMREGION_WRITEABLE;
+		rwx |= SBI_DOMAIN_MEMREGION_M_WRITABLE;
+
 	if (access_flags & SBI_DOMAIN_EXECUTE)
-		rwx |= SBI_DOMAIN_MEMREGION_EXECUTABLE;
+		rwx |= SBI_DOMAIN_MEMREGION_M_EXECUTABLE;
+
 	if (access_flags & SBI_DOMAIN_MMIO)
-		mmio = TRUE;
+		mmio = true;
 
 	sbi_domain_for_each_memregion(dom, reg) {
 		rflags = reg->flags;
-		if (mode == PRV_M && !(rflags & SBI_DOMAIN_MEMREGION_MMODE))
-			continue;
+		rrwx = (mode == PRV_M ?
+			(rflags & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK) :
+			(rflags & SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK)
+			>> SBI_DOMAIN_MEMREGION_SU_ACCESS_SHIFT);
 
 		rstart = reg->base;
 		rend = (reg->order < __riscv_xlen) ?
 			rstart + ((1UL << reg->order) - 1) : -1UL;
 		if (rstart <= addr && addr <= rend) {
-			rmmio = (rflags & SBI_DOMAIN_MEMREGION_MMIO) ? TRUE : FALSE;
+			rmmio = (rflags & SBI_DOMAIN_MEMREGION_MMIO) ? true : false;
 			if (mmio != rmmio)
-				return FALSE;
-			return ((rflags & rwx) == rwx) ? TRUE : FALSE;
+				return false;
+			return ((rrwx & rwx) == rwx) ? true : false;
 		}
 	}
 
-	return (mode == PRV_M) ? TRUE : FALSE;
+	return (mode == PRV_M) ? true : false;
 }
 
 /* Check if region complies with constraints */
 static bool is_region_valid(const struct sbi_domain_memregion *reg)
 {
 	if (reg->order < 3 || __riscv_xlen < reg->order)
-		return FALSE;
+		return false;
 
 	if (reg->order == __riscv_xlen && reg->base != 0)
-		return FALSE;
+		return false;
 
 	if (reg->order < __riscv_xlen && (reg->base & (BIT(reg->order) - 1)))
-		return FALSE;
+		return false;
 
-	return TRUE;
+	return true;
 }
 
 /** Check if regionA is sub-region of regionB */
@@ -168,20 +188,19 @@ static bool is_region_subset(const struct sbi_domain_memregion *regA,
 	    (regA_start < regB_end) &&
 	    (regB_start < regA_end) &&
 	    (regA_end <= regB_end))
-		return TRUE;
+		return true;
 
-	return FALSE;
+	return false;
 }
 
-/** Check if regionA conflicts regionB */
-static bool is_region_conflict(const struct sbi_domain_memregion *regA,
-				const struct sbi_domain_memregion *regB)
+/** Check if regionA can be replaced by regionB */
+static bool is_region_compatible(const struct sbi_domain_memregion *regA,
+				 const struct sbi_domain_memregion *regB)
 {
-	if ((is_region_subset(regA, regB) || is_region_subset(regB, regA)) &&
-	    regA->flags == regB->flags)
-		return TRUE;
+	if (is_region_subset(regA, regB) && regA->flags == regB->flags)
+		return true;
 
-	return FALSE;
+	return false;
 }
 
 /** Check if regionA should be placed before regionB */
@@ -189,21 +208,73 @@ static bool is_region_before(const struct sbi_domain_memregion *regA,
 			     const struct sbi_domain_memregion *regB)
 {
 	if (regA->order < regB->order)
-		return TRUE;
+		return true;
 
 	if ((regA->order == regB->order) &&
 	    (regA->base < regB->base))
-		return TRUE;
+		return true;
 
-	return FALSE;
+	return false;
 }
 
-static int sanitize_domain(const struct sbi_platform *plat,
-			   struct sbi_domain *dom)
+static const struct sbi_domain_memregion *find_region(
+						const struct sbi_domain *dom,
+						unsigned long addr)
+{
+	unsigned long rstart, rend;
+	struct sbi_domain_memregion *reg;
+
+	sbi_domain_for_each_memregion(dom, reg) {
+		rstart = reg->base;
+		rend = (reg->order < __riscv_xlen) ?
+			rstart + ((1UL << reg->order) - 1) : -1UL;
+		if (rstart <= addr && addr <= rend)
+			return reg;
+	}
+
+	return NULL;
+}
+
+static const struct sbi_domain_memregion *find_next_subset_region(
+				const struct sbi_domain *dom,
+				const struct sbi_domain_memregion *reg,
+				unsigned long addr)
+{
+	struct sbi_domain_memregion *sreg, *ret = NULL;
+
+	sbi_domain_for_each_memregion(dom, sreg) {
+		if (sreg == reg || (sreg->base <= addr) ||
+		    !is_region_subset(sreg, reg))
+			continue;
+
+		if (!ret || (sreg->base < ret->base) ||
+		    ((sreg->base == ret->base) && (sreg->order < ret->order)))
+			ret = sreg;
+	}
+
+	return ret;
+}
+
+static void swap_region(struct sbi_domain_memregion* reg1,
+			struct sbi_domain_memregion* reg2)
+{
+	struct sbi_domain_memregion treg;
+
+	sbi_memcpy(&treg, reg1, sizeof(treg));
+	sbi_memcpy(reg1, reg2, sizeof(treg));
+	sbi_memcpy(reg2, &treg, sizeof(treg));
+}
+
+static void clear_region(struct sbi_domain_memregion* reg)
+{
+	sbi_memset(reg, 0x0, sizeof(*reg));
+}
+
+static int sanitize_domain(struct sbi_domain *dom)
 {
 	u32 i, j, count;
-	bool have_fw_reg;
-	struct sbi_domain_memregion treg, *reg, *reg1;
+	bool is_covered;
+	struct sbi_domain_memregion *reg, *reg1;
 
 	/* Check possible HARTs */
 	if (!dom->possible_harts) {
@@ -211,13 +282,14 @@ static int sanitize_domain(const struct sbi_platform *plat,
 			   __func__, dom->name);
 		return SBI_EINVAL;
 	}
-	sbi_hartmask_for_each_hart(i, dom->possible_harts) {
-		if (sbi_platform_hart_invalid(plat, i)) {
+	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
+		if (!sbi_hartindex_valid(i)) {
 			sbi_printf("%s: %s possible HART mask has invalid "
-				   "hart %d\n", __func__, dom->name, i);
+				   "hart %d\n", __func__,
+				   dom->name, sbi_hartindex_to_hartid(i));
 			return SBI_EINVAL;
 		}
-	};
+	}
 
 	/* Check memory regions */
 	if (!dom->regions) {
@@ -235,17 +307,13 @@ static int sanitize_domain(const struct sbi_platform *plat,
 		}
 	}
 
-	/* Count memory regions and check presence of firmware region */
+	/* Count memory regions */
 	count = 0;
-	have_fw_reg = FALSE;
-	sbi_domain_for_each_memregion(dom, reg) {
-		if (reg->order == root_fw_region.order &&
-		    reg->base == root_fw_region.base &&
-		    reg->flags == root_fw_region.flags)
-			have_fw_reg = TRUE;
+	sbi_domain_for_each_memregion(dom, reg)
 		count++;
-	}
-	if (!have_fw_reg) {
+
+	/* Check presence of firmware regions */
+	if (!dom->fw_region_inited) {
 		sbi_printf("%s: %s does not have firmware region\n",
 			   __func__, dom->name);
 		return SBI_EINVAL;
@@ -257,25 +325,38 @@ static int sanitize_domain(const struct sbi_platform *plat,
 		for (j = i + 1; j < count; j++) {
 			reg1 = &dom->regions[j];
 
-			if (is_region_conflict(reg1, reg)) {
-				sbi_printf("%s: %s conflict between regions "
-					"(base=0x%lx order=%lu flags=0x%lx) and "
-					"(base=0x%lx order=%lu flags=0x%lx)\n",
-					__func__, dom->name,
-					reg->base, reg->order, reg->flags,
-					reg1->base, reg1->order, reg1->flags);
-				return SBI_EINVAL;
-			}
-
 			if (!is_region_before(reg1, reg))
 				continue;
 
-			sbi_memcpy(&treg, reg1, sizeof(treg));
-			sbi_memcpy(reg1, reg, sizeof(treg));
-			sbi_memcpy(reg, &treg, sizeof(treg));
+			swap_region(reg, reg1);
 		}
 	}
 
+	/* Remove covered regions */
+	while(i < (count - 1)) {
+		is_covered = false;
+		reg = &dom->regions[i];
+
+		for (j = i + 1; j < count; j++) {
+			reg1 = &dom->regions[j];
+
+			if (is_region_compatible(reg, reg1)) {
+				is_covered = true;
+				break;
+			}
+		}
+
+		/* find a region is superset of reg, remove reg */
+		if (is_covered) {
+			for (j = i; j < (count - 1); j++)
+				swap_region(&dom->regions[j],
+					    &dom->regions[j + 1]);
+			clear_region(&dom->regions[count - 1]);
+			count--;
+		} else
+			i++;
+	}
+
 	/*
 	 * We don't need to check boot HART id of domain because if boot
 	 * HART id is not possible/assigned to this domain then it won't
@@ -285,7 +366,7 @@ static int sanitize_domain(const struct sbi_platform *plat,
 	/*
 	 * Check next mode
 	 *
-	 * We only allow next mode to be S-mode or U-mode.so that we can
+	 * We only allow next mode to be S-mode or U-mode, so that we can
 	 * protect M-mode context and enforce checks on memory accesses.
 	 */
 	if (dom->next_mode != PRV_S &&
@@ -295,7 +376,7 @@ static int sanitize_domain(const struct sbi_platform *plat,
 		return SBI_EINVAL;
 	}
 
-	/* Check next address and next mode*/
+	/* Check next address and next mode */
 	if (!sbi_domain_check_addr(dom, dom->next_addr, dom->next_mode,
 				   SBI_DOMAIN_EXECUTE)) {
 		sbi_printf("%s: %s next booting stage address 0x%lx can't "
@@ -306,9 +387,40 @@ static int sanitize_domain(const struct sbi_platform *plat,
 	return 0;
 }
 
+bool sbi_domain_check_addr_range(const struct sbi_domain *dom,
+				 unsigned long addr, unsigned long size,
+				 unsigned long mode,
+				 unsigned long access_flags)
+{
+	unsigned long max = addr + size;
+	const struct sbi_domain_memregion *reg, *sreg;
+
+	if (!dom)
+		return false;
+
+	while (addr < max) {
+		reg = find_region(dom, addr);
+		if (!reg)
+			return false;
+
+		if (!sbi_domain_check_addr(dom, addr, mode, access_flags))
+			return false;
+
+		sreg = find_next_subset_region(dom, reg, addr);
+		if (sreg)
+			addr = sreg->base;
+		else if (reg->order < __riscv_xlen)
+			addr = reg->base + (1UL << reg->order);
+		else
+			break;
+	}
+
+	return true;
+}
+
 void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 {
-	u32 i, k;
+	u32 i, j, k;
 	unsigned long rstart, rend;
 	struct sbi_domain_memregion *reg;
 
@@ -320,9 +432,11 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 
 	k = 0;
 	sbi_printf("Domain%d HARTs       %s: ", dom->index, suffix);
-	sbi_hartmask_for_each_hart(i, dom->possible_harts)
+	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
+		j = sbi_hartindex_to_hartid(i);
 		sbi_printf("%s%d%s", (k++) ? "," : "",
-			   i, sbi_domain_is_assigned_hart(dom, i) ? "*" : "");
+			   j, sbi_domain_is_assigned_hart(dom, j) ? "*" : "");
+	}
 	sbi_printf("\n");
 
 	i = 0;
@@ -335,15 +449,25 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 			   dom->index, i, suffix, rstart, rend);
 
 		k = 0;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_MMODE)
-			sbi_printf("%cM", (k++) ? ',' : '(');
+
+		sbi_printf("M: ");
 		if (reg->flags & SBI_DOMAIN_MEMREGION_MMIO)
 			sbi_printf("%cI", (k++) ? ',' : '(');
-		if (reg->flags & SBI_DOMAIN_MEMREGION_READABLE)
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_READABLE)
 			sbi_printf("%cR", (k++) ? ',' : '(');
-		if (reg->flags & SBI_DOMAIN_MEMREGION_WRITEABLE)
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_WRITABLE)
 			sbi_printf("%cW", (k++) ? ',' : '(');
-		if (reg->flags & SBI_DOMAIN_MEMREGION_EXECUTABLE)
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_EXECUTABLE)
+			sbi_printf("%cX", (k++) ? ',' : '(');
+		sbi_printf("%s ", (k++) ? ")" : "()");
+
+		k = 0;
+		sbi_printf("S/U: ");
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_READABLE)
+			sbi_printf("%cR", (k++) ? ',' : '(');
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE)
+			sbi_printf("%cW", (k++) ? ',' : '(');
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
 			sbi_printf("%cX", (k++) ? ',' : '(');
 		sbi_printf("%s\n", (k++) ? ")" : "()");
 
@@ -370,10 +494,13 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 	default:
 		sbi_printf("Unknown\n");
 		break;
-	};
+	}
 
 	sbi_printf("Domain%d SysReset    %s: %s\n",
 		   dom->index, suffix, (dom->system_reset_allowed) ? "yes" : "no");
+
+	sbi_printf("Domain%d SysSuspend  %s: %s\n",
+		   dom->index, suffix, (dom->system_suspend_allowed) ? "yes" : "no");
 }
 
 void sbi_domain_dump_all(const char *suffix)
@@ -394,7 +521,6 @@ int sbi_domain_register(struct sbi_domain *dom,
 	int rc;
 	struct sbi_domain *tdom;
 	u32 cold_hartid = current_hartid();
-	const struct sbi_platform *plat = sbi_platform_thishart_ptr();
 
 	/* Sanity checks */
 	if (!dom || !assign_mask || domain_finalized)
@@ -417,7 +543,7 @@ int sbi_domain_register(struct sbi_domain *dom,
 	}
 
 	/* Sanitize discovered domain */
-	rc = sanitize_domain(plat, dom);
+	rc = sanitize_domain(dom);
 	if (rc) {
 		sbi_printf("%s: sanity checks failed for"
 			   " %s (error %d)\n", __func__,
@@ -433,22 +559,22 @@ int sbi_domain_register(struct sbi_domain *dom,
 	sbi_hartmask_clear_all(&dom->assigned_harts);
 
 	/* Assign domain to HART if HART is a possible HART */
-	sbi_hartmask_for_each_hart(i, assign_mask) {
-		if (!sbi_hartmask_test_hart(i, dom->possible_harts))
+	sbi_hartmask_for_each_hartindex(i, assign_mask) {
+		if (!sbi_hartmask_test_hartindex(i, dom->possible_harts))
 			continue;
 
-		tdom = hartid_to_domain_table[i];
+		tdom = sbi_hartindex_to_domain(i);
 		if (tdom)
-			sbi_hartmask_clear_hart(i,
+			sbi_hartmask_clear_hartindex(i,
 					&tdom->assigned_harts);
-		hartid_to_domain_table[i] = dom;
-		sbi_hartmask_set_hart(i, &dom->assigned_harts);
+		update_hartindex_to_domain(i, dom);
+		sbi_hartmask_set_hartindex(i, &dom->assigned_harts);
 
 		/*
 		 * If cold boot HART is assigned to this domain then
 		 * override boot HART of this domain.
 		 */
-		if (i == cold_hartid &&
+		if (sbi_hartindex_to_hartid(i) == cold_hartid &&
 		    dom->boot_hartid != cold_hartid) {
 			sbi_printf("Domain%d Boot HARTID forced to"
 				   " %d\n", dom->index, cold_hartid);
@@ -464,34 +590,28 @@ int sbi_domain_root_add_memregion(const struct sbi_domain_memregion *reg)
 	int rc;
 	bool reg_merged;
 	struct sbi_domain_memregion *nreg, *nreg1, *nreg2;
-	const struct sbi_platform *plat = sbi_platform_thishart_ptr();
 
 	/* Sanity checks */
-	if (!reg || domain_finalized ||
-	    (root.regions != root_memregs) ||
+	if (!reg || domain_finalized || !root.regions ||
 	    (ROOT_REGION_MAX <= root_memregs_count))
 		return SBI_EINVAL;
 
-	/* Check for conflicts */
+	/* Check whether compatible region exists for the new one */
 	sbi_domain_for_each_memregion(&root, nreg) {
-		if (is_region_conflict(reg, nreg)) {
-			sbi_printf("%s: is_region_conflict check failed"
-			" 0x%lx conflicts existing 0x%lx\n", __func__,
-				   reg->base, nreg->base);
-			return SBI_EALREADY;
-		}
+		if (is_region_compatible(reg, nreg))
+			return 0;
 	}
 
 	/* Append the memregion to root memregions */
-	nreg = &root_memregs[root_memregs_count];
+	nreg = &root.regions[root_memregs_count];
 	sbi_memcpy(nreg, reg, sizeof(*reg));
 	root_memregs_count++;
-	root_memregs[root_memregs_count].order = 0;
+	root.regions[root_memregs_count].order = 0;
 
 	/* Sort and optimize root regions */
 	do {
 		/* Sanitize the root domain so that memregions are sorted */
-		rc = sanitize_domain(plat, &root);
+		rc = sanitize_domain(&root);
 		if (rc) {
 			sbi_printf("%s: sanity checks failed for"
 				   " %s (error %d)\n", __func__,
@@ -569,36 +689,37 @@ int sbi_domain_finalize(struct sbi_scratch *scratch, u32 cold_hartid)
 
 	/* Startup boot HART of domains */
 	sbi_domain_for_each(i, dom) {
-		/* Domain boot HART */
-		dhart = dom->boot_hartid;
+		/* Domain boot HART index */
+		dhart = sbi_hartid_to_hartindex(dom->boot_hartid);
 
 		/* Ignore of boot HART is off limits */
-		if (SBI_HARTMASK_MAX_BITS <= dhart)
+		if (!sbi_hartindex_valid(dhart))
 			continue;
 
 		/* Ignore if boot HART not possible for this domain */
-		if (!sbi_hartmask_test_hart(dhart, dom->possible_harts))
+		if (!sbi_hartmask_test_hartindex(dhart, dom->possible_harts))
 			continue;
 
 		/* Ignore if boot HART assigned different domain */
-		if (sbi_hartid_to_domain(dhart) != dom ||
-		    !sbi_hartmask_test_hart(dhart, &dom->assigned_harts))
+		if (sbi_hartindex_to_domain(dhart) != dom ||
+		    !sbi_hartmask_test_hartindex(dhart, &dom->assigned_harts))
 			continue;
 
 		/* Startup boot HART of domain */
-		if (dhart == cold_hartid) {
+		if (dom->boot_hartid == cold_hartid) {
 			scratch->next_addr = dom->next_addr;
 			scratch->next_mode = dom->next_mode;
 			scratch->next_arg1 = dom->next_arg1;
 		} else {
-			rc = sbi_hsm_hart_start(scratch, NULL, dhart,
+			rc = sbi_hsm_hart_start(scratch, NULL,
+						dom->boot_hartid,
 						dom->next_addr,
 						dom->next_mode,
 						dom->next_arg1);
 			if (rc) {
 				sbi_printf("%s: failed to start boot HART %d"
 					   " for %s (error %d)\n", __func__,
-					   dhart, dom->name, rc);
+					   dom->boot_hartid, dom->name, rc);
 				return rc;
 			}
 		}
@@ -616,18 +737,69 @@ int sbi_domain_finalize(struct sbi_scratch *scratch, u32 cold_hartid)
 int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid)
 {
 	u32 i;
+	int rc;
+	struct sbi_hartmask *root_hmask;
+	struct sbi_domain_memregion *root_memregs;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	/* Root domain firmware memory region */
-	sbi_domain_memregion_init(scratch->fw_start, scratch->fw_size, 0,
-				  &root_fw_region);
-	domain_memregion_initfw(&root_memregs[root_memregs_count++]);
+	if (scratch->fw_rw_offset == 0 ||
+	    (scratch->fw_rw_offset & (scratch->fw_rw_offset - 1)) != 0) {
+		sbi_printf("%s: fw_rw_offset is not a power of 2 (0x%lx)\n",
+			   __func__, scratch->fw_rw_offset);
+		return SBI_EINVAL;
+	}
 
-	/* Root domain allow everything memory region */
+	if ((scratch->fw_start & (scratch->fw_rw_offset - 1)) != 0) {
+		sbi_printf("%s: fw_start and fw_rw_offset not aligned\n",
+			   __func__);
+		return SBI_EINVAL;
+	}
+
+	domain_hart_ptr_offset = sbi_scratch_alloc_type_offset(void *);
+	if (!domain_hart_ptr_offset)
+		return SBI_ENOMEM;
+
+	root_memregs = sbi_calloc(sizeof(*root_memregs), ROOT_REGION_MAX + 1);
+	if (!root_memregs) {
+		sbi_printf("%s: no memory for root regions\n", __func__);
+		rc = SBI_ENOMEM;
+		goto fail_free_domain_hart_ptr_offset;
+	}
+	root.regions = root_memregs;
+
+	root_hmask = sbi_zalloc(sizeof(*root_hmask));
+	if (!root_hmask) {
+		sbi_printf("%s: no memory for root hartmask\n", __func__);
+		rc = SBI_ENOMEM;
+		goto fail_free_root_memregs;
+	}
+	root.possible_harts = root_hmask;
+
+	/* Root domain firmware memory region */
+	sbi_domain_memregion_init(scratch->fw_start, scratch->fw_rw_offset,
+				  (SBI_DOMAIN_MEMREGION_M_READABLE |
+				   SBI_DOMAIN_MEMREGION_M_EXECUTABLE),
+				  &root_memregs[root_memregs_count++]);
+
+	sbi_domain_memregion_init((scratch->fw_start + scratch->fw_rw_offset),
+				  (scratch->fw_size - scratch->fw_rw_offset),
+				  (SBI_DOMAIN_MEMREGION_M_READABLE |
+				   SBI_DOMAIN_MEMREGION_M_WRITABLE),
+				  &root_memregs[root_memregs_count++]);
+
+	root.fw_region_inited = true;
+
+	/*
+	 * Allow SU RWX on rest of the memory region. Since pmp entries
+	 * have implicit priority on index, previous entries will
+	 * deny access to SU on M-mode region. Also, M-mode will not
+	 * have access to SU region while previous entries will allow
+	 * access to M-mode regions.
+	 */
 	sbi_domain_memregion_init(0, ~0UL,
-				  (SBI_DOMAIN_MEMREGION_READABLE |
-				   SBI_DOMAIN_MEMREGION_WRITEABLE |
-				   SBI_DOMAIN_MEMREGION_EXECUTABLE),
+				  (SBI_DOMAIN_MEMREGION_SU_READABLE |
+				   SBI_DOMAIN_MEMREGION_SU_WRITABLE |
+				   SBI_DOMAIN_MEMREGION_SU_EXECUTABLE),
 				  &root_memregs[root_memregs_count++]);
 
 	/* Root domain memory region end */
@@ -642,11 +814,21 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid)
 	root.next_mode = scratch->next_mode;
 
 	/* Root domain possible and assigned HARTs */
-	for (i = 0; i < SBI_HARTMASK_MAX_BITS; i++) {
-		if (sbi_platform_hart_invalid(plat, i))
-			continue;
-		sbi_hartmask_set_hart(i, &root_hmask);
-	}
+	for (i = 0; i < plat->hart_count; i++)
+		sbi_hartmask_set_hartindex(i, root_hmask);
 
-	return sbi_domain_register(&root, &root_hmask);
+	/* Finally register the root domain */
+	rc = sbi_domain_register(&root, root_hmask);
+	if (rc)
+		goto fail_free_root_hmask;
+
+	return 0;
+
+fail_free_root_hmask:
+	sbi_free(root_hmask);
+fail_free_root_memregs:
+	sbi_free(root_memregs);
+fail_free_domain_hart_ptr_offset:
+	sbi_scratch_free_offset(domain_hart_ptr_offset);
+	return rc;
 }

lib/sbi/sbi_ecall.c

@@ -78,7 +78,7 @@ int sbi_ecall_register_extension(struct sbi_ecall_extension *ext)
 
 void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext)
 {
-	bool found = FALSE;
+	bool found = false;
 	struct sbi_ecall_extension *t;
 
 	if (!ext)
@@ -86,7 +86,7 @@ void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext)
 
 	sbi_list_for_each_entry(t, &ecall_exts_list, head) {
 		if (t == ext) {
-			found = TRUE;
+			found = true;
 			break;
 		}
 	}
@@ -101,14 +101,12 @@ int sbi_ecall_handler(struct sbi_trap_regs *regs)
 	struct sbi_ecall_extension *ext;
 	unsigned long extension_id = regs->a7;
 	unsigned long func_id = regs->a6;
-	struct sbi_trap_info trap = {0};
-	unsigned long out_val = 0;
+	struct sbi_ecall_return out = {0};
 	bool is_0_1_spec = 0;
 
 	ext = sbi_ecall_find_extension(extension_id);
 	if (ext && ext->handle) {
-		ret = ext->handle(extension_id, func_id,
-				  regs, &out_val, &trap);
+		ret = ext->handle(extension_id, func_id, regs, &out);
 		if (extension_id >= SBI_EXT_0_1_SET_TIMER &&
 		    extension_id <= SBI_EXT_0_1_SHUTDOWN)
 			is_0_1_spec = 1;
@@ -116,11 +114,10 @@ int sbi_ecall_handler(struct sbi_trap_regs *regs)
 		ret = SBI_ENOTSUPP;
 	}
 
-	if (ret == SBI_ETRAP) {
-		trap.epc = regs->mepc;
-		sbi_trap_redirect(regs, &trap);
-	} else {
-		if (ret < SBI_LAST_ERR) {
+	if (!out.skip_regs_update) {
+		if (ret < SBI_LAST_ERR ||
+		    (extension_id != SBI_EXT_0_1_CONSOLE_GETCHAR &&
+		     SBI_SUCCESS < ret)) {
 			sbi_printf("%s: Invalid error %d for ext=0x%lx "
 				   "func=0x%lx\n", __func__, ret,
 				   extension_id, func_id);
@@ -138,7 +135,7 @@ int sbi_ecall_handler(struct sbi_trap_regs *regs)
 		regs->mepc += 4;
 		regs->a0 = ret;
 		if (!is_0_1_spec)
-			regs->a1 = out_val;
+			regs->a1 = out.value;
 	}
 
 	return 0;
@@ -152,7 +149,10 @@ int sbi_ecall_init(void)
 
 	for (i = 0; i < sbi_ecall_exts_size; i++) {
 		ext = sbi_ecall_exts[i];
-		ret = sbi_ecall_register_extension(ext);
+		ret = SBI_ENODEV;
+
+		if (ext->register_extensions)
+			ret = ext->register_extensions();
 		if (ret)
 			return ret;
 	}

lib/sbi/sbi_ecall_base.c

@@ -33,37 +33,36 @@ static int sbi_ecall_base_probe(unsigned long extid, unsigned long *out_val)
 }
 
 static int sbi_ecall_base_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	int ret = 0;
 
 	switch (funcid) {
 	case SBI_EXT_BASE_GET_SPEC_VERSION:
-		*out_val = (SBI_ECALL_VERSION_MAJOR <<
-			   SBI_SPEC_VERSION_MAJOR_OFFSET) &
-			   (SBI_SPEC_VERSION_MAJOR_MASK <<
-			    SBI_SPEC_VERSION_MAJOR_OFFSET);
-		*out_val = *out_val | SBI_ECALL_VERSION_MINOR;
+		out->value = (SBI_ECALL_VERSION_MAJOR <<
+			      SBI_SPEC_VERSION_MAJOR_OFFSET) &
+			     (SBI_SPEC_VERSION_MAJOR_MASK <<
+			      SBI_SPEC_VERSION_MAJOR_OFFSET);
+		out->value = out->value | SBI_ECALL_VERSION_MINOR;
 		break;
 	case SBI_EXT_BASE_GET_IMP_ID:
-		*out_val = sbi_ecall_get_impid();
+		out->value = sbi_ecall_get_impid();
 		break;
 	case SBI_EXT_BASE_GET_IMP_VERSION:
-		*out_val = OPENSBI_VERSION;
+		out->value = OPENSBI_VERSION;
 		break;
 	case SBI_EXT_BASE_GET_MVENDORID:
-		*out_val = csr_read(CSR_MVENDORID);
+		out->value = csr_read(CSR_MVENDORID);
 		break;
 	case SBI_EXT_BASE_GET_MARCHID:
-		*out_val = csr_read(CSR_MARCHID);
+		out->value = csr_read(CSR_MARCHID);
 		break;
 	case SBI_EXT_BASE_GET_MIMPID:
-		*out_val = csr_read(CSR_MIMPID);
+		out->value = csr_read(CSR_MIMPID);
 		break;
 	case SBI_EXT_BASE_PROBE_EXT:
-		ret = sbi_ecall_base_probe(regs->a0, out_val);
+		ret = sbi_ecall_base_probe(regs->a0, &out->value);
 		break;
 	default:
 		ret = SBI_ENOTSUPP;
@@ -72,8 +71,16 @@ static int sbi_ecall_base_handler(unsigned long extid, unsigned long funcid,
 	return ret;
 }
 
+struct sbi_ecall_extension ecall_base;
+
+static int sbi_ecall_base_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_base);
+}
+
 struct sbi_ecall_extension ecall_base = {
-	.extid_start = SBI_EXT_BASE,
-	.extid_end = SBI_EXT_BASE,
-	.handle = sbi_ecall_base_handler,
+	.extid_start		= SBI_EXT_BASE,
+	.extid_end		= SBI_EXT_BASE,
+	.register_extensions	= sbi_ecall_base_register_extensions,
+	.handle			= sbi_ecall_base_handler,
 };

lib/sbi/sbi_ecall_cppc.c

@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ */
+
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_cppc.h>
+
+static int sbi_ecall_cppc_handler(unsigned long extid, unsigned long funcid,
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
+{
+	int ret = 0;
+	uint64_t temp;
+
+	switch (funcid) {
+	case SBI_EXT_CPPC_READ:
+		ret = sbi_cppc_read(regs->a0, &temp);
+		out->value = temp;
+		break;
+	case SBI_EXT_CPPC_READ_HI:
+#if __riscv_xlen == 32
+		ret = sbi_cppc_read(regs->a0, &temp);
+		out->value = temp >> 32;
+#else
+		out->value = 0;
+#endif
+		break;
+	case SBI_EXT_CPPC_WRITE:
+		ret = sbi_cppc_write(regs->a0, regs->a1);
+		break;
+	case SBI_EXT_CPPC_PROBE:
+		ret = sbi_cppc_probe(regs->a0);
+		if (ret >= 0) {
+			out->value = ret;
+			ret = 0;
+		}
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+	}
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_cppc;
+
+static int sbi_ecall_cppc_register_extensions(void)
+{
+	if (!sbi_cppc_get_device())
+		return 0;
+
+	return sbi_ecall_register_extension(&ecall_cppc);
+}
+
+struct sbi_ecall_extension ecall_cppc = {
+	.extid_start		= SBI_EXT_CPPC,
+	.extid_end		= SBI_EXT_CPPC,
+	.register_extensions	= sbi_ecall_cppc_register_extensions,
+	.handle			= sbi_ecall_cppc_handler,
+};

lib/sbi/sbi_ecall_dbcn.c

@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2022 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_hart.h>
+
+static int sbi_ecall_dbcn_handler(unsigned long extid, unsigned long funcid,
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
+{
+	ulong smode = (csr_read(CSR_MSTATUS) & MSTATUS_MPP) >>
+			MSTATUS_MPP_SHIFT;
+
+	switch (funcid) {
+	case SBI_EXT_DBCN_CONSOLE_WRITE:
+	case SBI_EXT_DBCN_CONSOLE_READ:
+		/*
+		 * On RV32, the M-mode can only access the first 4GB of
+		 * the physical address space because M-mode does not have
+		 * MMU to access full 34-bit physical address space.
+		 *
+		 * Based on above, we simply fail if the upper 32bits of
+		 * the physical address (i.e. a2 register) is non-zero on
+		 * RV32.
+		 *
+		 * Analogously, we fail if the upper 64bit of the
+		 * physical address (i.e. a2 register) is non-zero on
+		 * RV64.
+		 */
+		if (regs->a2)
+			return SBI_ERR_FAILED;
+
+		if (!sbi_domain_check_addr_range(sbi_domain_thishart_ptr(),
+					regs->a1, regs->a0, smode,
+					SBI_DOMAIN_READ|SBI_DOMAIN_WRITE))
+			return SBI_ERR_INVALID_PARAM;
+		sbi_hart_map_saddr(regs->a1, regs->a0);
+		if (funcid == SBI_EXT_DBCN_CONSOLE_WRITE)
+			out->value = sbi_nputs((const char *)regs->a1, regs->a0);
+		else
+			out->value = sbi_ngets((char *)regs->a1, regs->a0);
+		sbi_hart_unmap_saddr();
+		return 0;
+	case SBI_EXT_DBCN_CONSOLE_WRITE_BYTE:
+		sbi_putc(regs->a0);
+		return 0;
+	default:
+		break;
+	}
+
+	return SBI_ENOTSUPP;
+}
+
+struct sbi_ecall_extension ecall_dbcn;
+
+static int sbi_ecall_dbcn_register_extensions(void)
+{
+	if (!sbi_console_get_device())
+		return 0;
+
+	return sbi_ecall_register_extension(&ecall_dbcn);
+}
+
+struct sbi_ecall_extension ecall_dbcn = {
+	.extid_start		= SBI_EXT_DBCN,
+	.extid_end		= SBI_EXT_DBCN,
+	.register_extensions	= sbi_ecall_dbcn_register_extensions,
+	.handle			= sbi_ecall_dbcn_handler,
+};

lib/sbi/sbi_ecall_hsm.c

@@ -12,15 +12,13 @@
 #include <sbi/sbi_ecall_interface.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_trap.h>
-#include <sbi/sbi_version.h>
 #include <sbi/sbi_hsm.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/riscv_asm.h>
 
 static int sbi_ecall_hsm_handler(unsigned long extid, unsigned long funcid,
-				 const struct sbi_trap_regs *regs,
-				 unsigned long *out_val,
-				 struct sbi_trap_info *out_trap)
+				 struct sbi_trap_regs *regs,
+				 struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
@@ -33,7 +31,7 @@ static int sbi_ecall_hsm_handler(unsigned long extid, unsigned long funcid,
 					 regs->a0, regs->a1, smode, regs->a2);
 		break;
 	case SBI_EXT_HSM_HART_STOP:
-		ret = sbi_hsm_hart_stop(scratch, TRUE);
+		ret = sbi_hsm_hart_stop(scratch, true);
 		break;
 	case SBI_EXT_HSM_HART_GET_STATUS:
 		ret = sbi_hsm_hart_get_state(sbi_domain_thishart_ptr(),
@@ -45,17 +43,26 @@ static int sbi_ecall_hsm_handler(unsigned long extid, unsigned long funcid,
 		break;
 	default:
 		ret = SBI_ENOTSUPP;
-	};
+	}
+
 	if (ret >= 0) {
-		*out_val = ret;
+		out->value = ret;
 		ret = 0;
 	}
 
 	return ret;
 }
 
+struct sbi_ecall_extension ecall_hsm;
+
+static int sbi_ecall_hsm_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_hsm);
+}
+
 struct sbi_ecall_extension ecall_hsm = {
-	.extid_start = SBI_EXT_HSM,
-	.extid_end = SBI_EXT_HSM,
-	.handle = sbi_ecall_hsm_handler,
+	.extid_start		= SBI_EXT_HSM,
+	.extid_end		= SBI_EXT_HSM,
+	.register_extensions	= sbi_ecall_hsm_register_extensions,
+	.handle			= sbi_ecall_hsm_handler,
 };

lib/sbi/sbi_ecall_ipi.c

@@ -15,9 +15,8 @@
 #include <sbi/sbi_ipi.h>
 
 static int sbi_ecall_ipi_handler(unsigned long extid, unsigned long funcid,
-				 const struct sbi_trap_regs *regs,
-				 unsigned long *out_val,
-				 struct sbi_trap_info *out_trap)
+				 struct sbi_trap_regs *regs,
+				 struct sbi_ecall_return *out)
 {
 	int ret = 0;
 
@@ -29,8 +28,16 @@ static int sbi_ecall_ipi_handler(unsigned long extid, unsigned long funcid,
 	return ret;
 }
 
+struct sbi_ecall_extension ecall_ipi;
+
+static int sbi_ecall_ipi_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_ipi);
+}
+
 struct sbi_ecall_extension ecall_ipi = {
-	.extid_start = SBI_EXT_IPI,
-	.extid_end = SBI_EXT_IPI,
-	.handle = sbi_ecall_ipi_handler,
+	.extid_start		= SBI_EXT_IPI,
+	.extid_end		= SBI_EXT_IPI,
+	.register_extensions	= sbi_ecall_ipi_register_extensions,
+	.handle			= sbi_ecall_ipi_handler,
 };

lib/sbi/sbi_ecall_legacy.c

@@ -24,32 +24,32 @@
 #include <sbi/sbi_unpriv.h>
 #include <sbi/sbi_hart.h>
 
-static int sbi_load_hart_mask_unpriv(ulong *pmask, ulong *hmask,
-				     struct sbi_trap_info *uptrap)
+static bool sbi_load_hart_mask_unpriv(ulong *pmask, ulong *hmask,
+				      struct sbi_trap_info *uptrap)
 {
 	ulong mask = 0;
 
 	if (pmask) {
 		mask = sbi_load_ulong(pmask, uptrap);
 		if (uptrap->cause)
-			return SBI_ETRAP;
+			return false;
 	} else {
 		sbi_hsm_hart_interruptible_mask(sbi_domain_thishart_ptr(),
 						0, &mask);
 	}
 	*hmask = mask;
 
-	return 0;
+	return true;
 }
 
 static int sbi_ecall_legacy_handler(unsigned long extid, unsigned long funcid,
-				    const struct sbi_trap_regs *regs,
-				    unsigned long *out_val,
-				    struct sbi_trap_info *out_trap)
+				    struct sbi_trap_regs *regs,
+				    struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	struct sbi_tlb_info tlb_info;
 	u32 source_hart = current_hartid();
+	struct sbi_trap_info trap = {0};
 	ulong hmask = 0;
 
 	switch (extid) {
@@ -70,40 +70,51 @@ static int sbi_ecall_legacy_handler(unsigned long extid, unsigned long funcid,
 		sbi_ipi_clear_smode();
 		break;
 	case SBI_EXT_0_1_SEND_IPI:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP)
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			ret = sbi_ipi_send_smode(hmask, 0);
+		} else {
+			trap.epc = regs->mepc;
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
+		}
 		break;
 	case SBI_EXT_0_1_REMOTE_FENCE_I:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP) {
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			SBI_TLB_INFO_INIT(&tlb_info, 0, 0, 0, 0,
-					  sbi_tlb_local_fence_i,
-					  source_hart);
+					  SBI_TLB_FENCE_I, source_hart);
 			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		} else {
+			trap.epc = regs->mepc;
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
 		}
 		break;
 	case SBI_EXT_0_1_REMOTE_SFENCE_VMA:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP) {
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			SBI_TLB_INFO_INIT(&tlb_info, regs->a1, regs->a2, 0, 0,
-					  sbi_tlb_local_sfence_vma,
-					  source_hart);
+					  SBI_TLB_SFENCE_VMA, source_hart);
 			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		} else {
+			trap.epc = regs->mepc;
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
 		}
 		break;
 	case SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP) {
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			SBI_TLB_INFO_INIT(&tlb_info, regs->a1,
 					  regs->a2, regs->a3, 0,
-					  sbi_tlb_local_sfence_vma_asid,
+					  SBI_TLB_SFENCE_VMA_ASID,
 					  source_hart);
 			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		} else {
+			trap.epc = regs->mepc;
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
 		}
 		break;
 	case SBI_EXT_0_1_SHUTDOWN:
@@ -112,13 +123,21 @@ static int sbi_ecall_legacy_handler(unsigned long extid, unsigned long funcid,
 		break;
 	default:
 		ret = SBI_ENOTSUPP;
-	};
+	}
 
 	return ret;
 }
 
+struct sbi_ecall_extension ecall_legacy;
+
+static int sbi_ecall_legacy_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_legacy);
+}
+
 struct sbi_ecall_extension ecall_legacy = {
-	.extid_start = SBI_EXT_0_1_SET_TIMER,
-	.extid_end = SBI_EXT_0_1_SHUTDOWN,
-	.handle = sbi_ecall_legacy_handler,
+	.extid_start		= SBI_EXT_0_1_SET_TIMER,
+	.extid_end		= SBI_EXT_0_1_SHUTDOWN,
+	.register_extensions	= sbi_ecall_legacy_register_extensions,
+	.handle			= sbi_ecall_legacy_handler,
 };

lib/sbi/sbi_ecall_pmu.c

@@ -18,9 +18,8 @@
 #include <sbi/riscv_asm.h>
 
 static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
-				 const struct sbi_trap_regs *regs,
-				 unsigned long *out_val,
-				 struct sbi_trap_info *out_trap)
+				 struct sbi_trap_regs *regs,
+				 struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	uint64_t temp;
@@ -29,12 +28,12 @@ static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
 	case SBI_EXT_PMU_NUM_COUNTERS:
 		ret = sbi_pmu_num_ctr();
 		if (ret >= 0) {
-			*out_val = ret;
+			out->value = ret;
 			ret = 0;
 		}
 		break;
 	case SBI_EXT_PMU_COUNTER_GET_INFO:
-		ret = sbi_pmu_ctr_get_info(regs->a0, out_val);
+		ret = sbi_pmu_ctr_get_info(regs->a0, &out->value);
 		break;
 	case SBI_EXT_PMU_COUNTER_CFG_MATCH:
 #if __riscv_xlen == 32
@@ -45,14 +44,22 @@ static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
 		ret = sbi_pmu_ctr_cfg_match(regs->a0, regs->a1, regs->a2,
 					    regs->a3, temp);
 		if (ret >= 0) {
-			*out_val = ret;
+			out->value = ret;
 			ret = 0;
 		}
 
 		break;
 	case SBI_EXT_PMU_COUNTER_FW_READ:
 		ret = sbi_pmu_ctr_fw_read(regs->a0, &temp);
-		*out_val = temp;
+		out->value = temp;
+		break;
+	case SBI_EXT_PMU_COUNTER_FW_READ_HI:
+#if __riscv_xlen == 32
+		ret = sbi_pmu_ctr_fw_read(regs->a0, &temp);
+		out->value = temp >> 32;
+#else
+		out->value = 0;
+#endif
 		break;
 	case SBI_EXT_PMU_COUNTER_START:
 
@@ -66,23 +73,25 @@ static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
 	case SBI_EXT_PMU_COUNTER_STOP:
 		ret = sbi_pmu_ctr_stop(regs->a0, regs->a1, regs->a2);
 		break;
+	case SBI_EXT_PMU_SNAPSHOT_SET_SHMEM:
+		/* fallthrough as OpenSBI doesn't support snapshot yet */
 	default:
 		ret = SBI_ENOTSUPP;
-	};
+	}
 
 	return ret;
 }
 
-static int sbi_ecall_pmu_probe(unsigned long extid, unsigned long *out_val)
+struct sbi_ecall_extension ecall_pmu;
+
+static int sbi_ecall_pmu_register_extensions(void)
 {
-	/* PMU extension is always enabled */
-	*out_val = 1;
-	return 0;
+	return sbi_ecall_register_extension(&ecall_pmu);
 }
 
 struct sbi_ecall_extension ecall_pmu = {
-	.extid_start = SBI_EXT_PMU,
-	.extid_end = SBI_EXT_PMU,
-	.handle = sbi_ecall_pmu_handler,
-	.probe = sbi_ecall_pmu_probe,
+	.extid_start		= SBI_EXT_PMU,
+	.extid_end		= SBI_EXT_PMU,
+	.register_extensions	= sbi_ecall_pmu_register_extensions,
+	.handle			= sbi_ecall_pmu_handler,
 };

lib/sbi/sbi_ecall_rfence.c

@@ -16,9 +16,8 @@
 #include <sbi/sbi_tlb.h>
 
 static int sbi_ecall_rfence_handler(unsigned long extid, unsigned long funcid,
-				    const struct sbi_trap_regs *regs,
-				    unsigned long *out_val,
-				    struct sbi_trap_info *out_trap)
+				    struct sbi_trap_regs *regs,
+				    struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	unsigned long vmid;
@@ -33,54 +32,60 @@ static int sbi_ecall_rfence_handler(unsigned long extid, unsigned long funcid,
 	switch (funcid) {
 	case SBI_EXT_RFENCE_REMOTE_FENCE_I:
 		SBI_TLB_INFO_INIT(&tlb_info, 0, 0, 0, 0,
-				  sbi_tlb_local_fence_i, source_hart);
+				  SBI_TLB_FENCE_I, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, 0,
-				  sbi_tlb_local_hfence_gvma, source_hart);
+				  SBI_TLB_HFENCE_GVMA, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, regs->a4,
-				  sbi_tlb_local_hfence_gvma_vmid,
-				  source_hart);
+				  SBI_TLB_HFENCE_GVMA_VMID, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
 		vmid = (csr_read(CSR_HGATP) & HGATP_VMID_MASK);
 		vmid = vmid >> HGATP_VMID_SHIFT;
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, vmid,
-				  sbi_tlb_local_hfence_vvma, source_hart);
+				  SBI_TLB_HFENCE_VVMA, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
 		vmid = (csr_read(CSR_HGATP) & HGATP_VMID_MASK);
 		vmid = vmid >> HGATP_VMID_SHIFT;
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, regs->a4,
-				  vmid, sbi_tlb_local_hfence_vvma_asid,
-				  source_hart);
+				  vmid, SBI_TLB_HFENCE_VVMA_ASID, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, 0,
-				  sbi_tlb_local_sfence_vma, source_hart);
+				  SBI_TLB_SFENCE_VMA, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, regs->a4, 0,
-				  sbi_tlb_local_sfence_vma_asid, source_hart);
+				  SBI_TLB_SFENCE_VMA_ASID, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	default:
 		ret = SBI_ENOTSUPP;
-	};
+	}
 
 	return ret;
 }
 
+struct sbi_ecall_extension ecall_rfence;
+
+static int sbi_ecall_rfence_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_rfence);
+}
+
 struct sbi_ecall_extension ecall_rfence = {
-	.extid_start = SBI_EXT_RFENCE,
-	.extid_end = SBI_EXT_RFENCE,
-	.handle = sbi_ecall_rfence_handler,
+	.extid_start		= SBI_EXT_RFENCE,
+	.extid_end		= SBI_EXT_RFENCE,
+	.register_extensions	= sbi_ecall_rfence_register_extensions,
+	.handle			= sbi_ecall_rfence_handler,
 };

lib/sbi/sbi_ecall_srst.c

@@ -15,9 +15,8 @@
 #include <sbi/sbi_system.h>
 
 static int sbi_ecall_srst_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	if (funcid == SBI_EXT_SRST_RESET) {
 		if ((((u32)-1U) <= ((u64)regs->a0)) ||
@@ -48,28 +47,36 @@ static int sbi_ecall_srst_handler(unsigned long extid, unsigned long funcid,
 	return SBI_ENOTSUPP;
 }
 
-static int sbi_ecall_srst_probe(unsigned long extid, unsigned long *out_val)
+static bool srst_available(void)
 {
-	u32 type, count = 0;
+	u32 type;
 
 	/*
 	 * At least one standard reset types should be supported by
 	 * the platform for SBI SRST extension to be usable.
 	 */
-
 	for (type = 0; type <= SBI_SRST_RESET_TYPE_LAST; type++) {
 		if (sbi_system_reset_supported(type,
 					SBI_SRST_RESET_REASON_NONE))
-			count++;
+			return true;
 	}
 
-	*out_val = (count) ? 1 : 0;
-	return 0;
+	return false;
+}
+
+struct sbi_ecall_extension ecall_srst;
+
+static int sbi_ecall_srst_register_extensions(void)
+{
+	if (!srst_available())
+		return 0;
+
+	return sbi_ecall_register_extension(&ecall_srst);
 }
 
 struct sbi_ecall_extension ecall_srst = {
-	.extid_start = SBI_EXT_SRST,
-	.extid_end = SBI_EXT_SRST,
-	.handle = sbi_ecall_srst_handler,
-	.probe = sbi_ecall_srst_probe,
+	.extid_start		= SBI_EXT_SRST,
+	.extid_end		= SBI_EXT_SRST,
+	.register_extensions	= sbi_ecall_srst_register_extensions,
+	.handle			= sbi_ecall_srst_handler,
 };

lib/sbi/sbi_ecall_susp.c

@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: BSD-2-Clause
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_system.h>
+
+static int sbi_ecall_susp_handler(unsigned long extid, unsigned long funcid,
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
+{
+	int ret = SBI_ENOTSUPP;
+
+	if (funcid == SBI_EXT_SUSP_SUSPEND)
+		ret = sbi_system_suspend(regs->a0, regs->a1, regs->a2);
+
+	if (ret >= 0) {
+		out->value = ret;
+		ret = 0;
+	}
+
+	return ret;
+}
+
+static bool susp_available(void)
+{
+	u32 type;
+
+	/*
+	 * At least one suspend type should be supported by the
+	 * platform for the SBI SUSP extension to be usable.
+	 */
+	for (type = 0; type <= SBI_SUSP_SLEEP_TYPE_LAST; type++) {
+		if (sbi_system_suspend_supported(type))
+			return true;
+	}
+
+	return false;
+}
+
+struct sbi_ecall_extension ecall_susp;
+
+static int sbi_ecall_susp_register_extensions(void)
+{
+	if (!susp_available())
+		return 0;
+
+	return sbi_ecall_register_extension(&ecall_susp);
+}
+
+struct sbi_ecall_extension ecall_susp = {
+	.extid_start		= SBI_EXT_SUSP,
+	.extid_end		= SBI_EXT_SUSP,
+	.register_extensions	= sbi_ecall_susp_register_extensions,
+	.handle			= sbi_ecall_susp_handler,
+};

lib/sbi/sbi_ecall_time.c

@@ -15,9 +15,8 @@
 #include <sbi/sbi_timer.h>
 
 static int sbi_ecall_time_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	int ret = 0;
 
@@ -33,8 +32,16 @@ static int sbi_ecall_time_handler(unsigned long extid, unsigned long funcid,
 	return ret;
 }
 
+struct sbi_ecall_extension ecall_time;
+
+static int sbi_ecall_time_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_time);
+}
+
 struct sbi_ecall_extension ecall_time = {
-	.extid_start = SBI_EXT_TIME,
-	.extid_end = SBI_EXT_TIME,
-	.handle = sbi_ecall_time_handler,
+	.extid_start		= SBI_EXT_TIME,
+	.extid_end		= SBI_EXT_TIME,
+	.register_extensions	= sbi_ecall_time_register_extensions,
+	.handle			= sbi_ecall_time_handler,
 };

lib/sbi/sbi_ecall_vendor.c

@@ -13,28 +13,41 @@
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_trap.h>
+#include <sbi/riscv_asm.h>
 
-static int sbi_ecall_vendor_probe(unsigned long extid,
-				  unsigned long *out_val)
+static inline unsigned long sbi_ecall_vendor_id(void)
 {
-	*out_val = sbi_platform_vendor_ext_check(sbi_platform_thishart_ptr(),
-						 extid);
-	return 0;
+	return SBI_EXT_VENDOR_START +
+		(csr_read(CSR_MVENDORID) &
+		 (SBI_EXT_VENDOR_END - SBI_EXT_VENDOR_START));
 }
 
 static int sbi_ecall_vendor_handler(unsigned long extid, unsigned long funcid,
-				    const struct sbi_trap_regs *regs,
-				    unsigned long *out_val,
-				    struct sbi_trap_info *out_trap)
+				    struct sbi_trap_regs *regs,
+				    struct sbi_ecall_return *out)
 {
 	return sbi_platform_vendor_ext_provider(sbi_platform_thishart_ptr(),
-						extid, funcid, regs,
-						out_val, out_trap);
+						funcid, regs, out);
+}
+
+struct sbi_ecall_extension ecall_vendor;
+
+static int sbi_ecall_vendor_register_extensions(void)
+{
+	unsigned long extid = sbi_ecall_vendor_id();
+
+	if (!sbi_platform_vendor_ext_check(sbi_platform_thishart_ptr()))
+		return 0;
+
+	ecall_vendor.extid_start = extid;
+	ecall_vendor.extid_end = extid;
+
+	return sbi_ecall_register_extension(&ecall_vendor);
 }
 
 struct sbi_ecall_extension ecall_vendor = {
-	.extid_start = SBI_EXT_VENDOR_START,
-	.extid_end = SBI_EXT_VENDOR_END,
-	.probe = sbi_ecall_vendor_probe,
-	.handle = sbi_ecall_vendor_handler,
+	.extid_start		= SBI_EXT_VENDOR_START,
+	.extid_end		= SBI_EXT_VENDOR_END,
+	.register_extensions	= sbi_ecall_vendor_register_extensions,
+	.handle			= sbi_ecall_vendor_handler,
 };

lib/sbi/sbi_emulate_csr.c

@@ -39,7 +39,7 @@ static bool hpm_allowed(int hpm_num, ulong prev_mode, bool virt)
 			cen = 0;
 	}
 
-	return ((cen >> hpm_num) & 1) ? TRUE : FALSE;
+	return ((cen >> hpm_num) & 1) ? true : false;
 }
 
 int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs,
@@ -49,9 +49,9 @@ int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs,
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 	ulong prev_mode = (regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT;
 #if __riscv_xlen == 32
-	bool virt = (regs->mstatusH & MSTATUSH_MPV) ? TRUE : FALSE;
+	bool virt = (regs->mstatusH & MSTATUSH_MPV) ? true : false;
 #else
-	bool virt = (regs->mstatus & MSTATUS_MPV) ? TRUE : FALSE;
+	bool virt = (regs->mstatus & MSTATUS_MPV) ? true : false;
 #endif
 
 	switch (csr_num) {
@@ -109,7 +109,7 @@ int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs,
 
 #define switchcase_hpm(__uref, __mref, __csr)				\
 	case __csr:							\
-		if ((sbi_hart_mhpm_count(scratch) + 3) <= (__csr - __uref))\
+		if (sbi_hart_mhpm_mask(scratch) & (1 << (__csr - __uref)))\
 			return SBI_ENOTSUPP;				\
 		if (!hpm_allowed(__csr - __uref, prev_mode, virt))	\
 			return SBI_ENOTSUPP;				\
@@ -149,7 +149,7 @@ int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs,
 	default:
 		ret = SBI_ENOTSUPP;
 		break;
-	};
+	}
 
 	if (ret)
 		sbi_dprintf("%s: hartid%d: invalid csr_num=0x%x\n",
@@ -164,9 +164,9 @@ int sbi_emulate_csr_write(int csr_num, struct sbi_trap_regs *regs,
 	int ret = 0;
 	ulong prev_mode = (regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT;
 #if __riscv_xlen == 32
-	bool virt = (regs->mstatusH & MSTATUSH_MPV) ? TRUE : FALSE;
+	bool virt = (regs->mstatusH & MSTATUSH_MPV) ? true : false;
 #else
-	bool virt = (regs->mstatus & MSTATUS_MPV) ? TRUE : FALSE;
+	bool virt = (regs->mstatus & MSTATUS_MPV) ? true : false;
 #endif
 
 	switch (csr_num) {
@@ -187,7 +187,7 @@ int sbi_emulate_csr_write(int csr_num, struct sbi_trap_regs *regs,
 	default:
 		ret = SBI_ENOTSUPP;
 		break;
-	};
+	}
 
 	if (ret)
 		sbi_dprintf("%s: hartid%d: invalid csr_num=0x%x\n",

lib/sbi/sbi_fifo.c

@@ -26,7 +26,7 @@ void sbi_fifo_init(struct sbi_fifo *fifo, void *queue_mem, u16 entries,
 /* Note: must be called with fifo->qlock held */
 static inline bool __sbi_fifo_is_full(struct sbi_fifo *fifo)
 {
-	return (fifo->avail == fifo->num_entries) ? TRUE : FALSE;
+	return (fifo->avail == fifo->num_entries) ? true : false;
 }
 
 u16 sbi_fifo_avail(struct sbi_fifo *fifo)
@@ -75,7 +75,7 @@ static inline void  __sbi_fifo_enqueue(struct sbi_fifo *fifo, void *data)
 /* Note: must be called with fifo->qlock held */
 static inline bool __sbi_fifo_is_empty(struct sbi_fifo *fifo)
 {
-	return (fifo->avail == 0) ? TRUE : FALSE;
+	return (fifo->avail == 0) ? true : false;
 }
 
 int sbi_fifo_is_empty(struct sbi_fifo *fifo)
@@ -105,13 +105,13 @@ static inline void __sbi_fifo_reset(struct sbi_fifo *fifo)
 bool sbi_fifo_reset(struct sbi_fifo *fifo)
 {
 	if (!fifo)
-		return FALSE;
+		return false;
 
 	spin_lock(&fifo->qlock);
 	__sbi_fifo_reset(fifo);
 	spin_unlock(&fifo->qlock);
 
-	return TRUE;
+	return true;
 }
 
 /**

lib/sbi/sbi_hart.c

@@ -33,11 +33,11 @@ static unsigned long hart_features_offset;
 
 static void mstatus_init(struct sbi_scratch *scratch)
 {
-	unsigned long menvcfg_val, mstatus_val = 0;
 	int cidx;
-	unsigned int num_mhpm = sbi_hart_mhpm_count(scratch);
+	unsigned long mstatus_val = 0;
+	unsigned int mhpm_mask = sbi_hart_mhpm_mask(scratch);
 	uint64_t mhpmevent_init_val = 0;
-	uint64_t mstateen_val;
+	uint64_t menvcfg_val, mstateen_val;
 
 	/* Enable FPU */
 	if (misa_extension('D') || misa_extension('F'))
@@ -69,13 +69,14 @@ static void mstatus_init(struct sbi_scratch *scratch)
 	/**
 	 * The mhpmeventn[h] CSR should be initialized with interrupt disabled
 	 * and inhibited running in M-mode during init.
-	 * To keep it simple, only contiguous mhpmcounters are supported as a
-	 * platform with discontiguous mhpmcounters may not make much sense.
 	 */
 	mhpmevent_init_val |= (MHPMEVENT_OF | MHPMEVENT_MINH);
-	for (cidx = 0; cidx < num_mhpm; cidx++) {
+	for (cidx = 0; cidx <= 28; cidx++) {
+		if (!(mhpm_mask & 1 << (cidx + 3)))
+			continue;
 #if __riscv_xlen == 32
-		csr_write_num(CSR_MHPMEVENT3 + cidx, mhpmevent_init_val & 0xFFFFFFFF);
+		csr_write_num(CSR_MHPMEVENT3 + cidx,
+			       mhpmevent_init_val & 0xFFFFFFFF);
 		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
 			csr_write_num(CSR_MHPMEVENT3H + cidx,
 				      mhpmevent_init_val >> BITS_PER_LONG);
@@ -90,6 +91,7 @@ static void mstatus_init(struct sbi_scratch *scratch)
 		mstateen_val |= ((uint64_t)csr_read(CSR_MSTATEEN0H)) << 32;
 #endif
 		mstateen_val |= SMSTATEEN_STATEN;
+		mstateen_val |= SMSTATEEN0_CONTEXT;
 		mstateen_val |= SMSTATEEN0_HSENVCFG;
 
 		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMAIA))
@@ -106,58 +108,40 @@ static void mstatus_init(struct sbi_scratch *scratch)
 
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_12) {
 		menvcfg_val = csr_read(CSR_MENVCFG);
-
-		/*
-		 * Set menvcfg.CBZE == 1
-		 *
-		 * If Zicboz extension is not available then writes to
-		 * menvcfg.CBZE will be ignored because it is a WARL field.
-		 */
-		menvcfg_val |= ENVCFG_CBZE;
-
-		/*
-		 * Set menvcfg.CBCFE == 1
-		 *
-		 * If Zicbom extension is not available then writes to
-		 * menvcfg.CBCFE will be ignored because it is a WARL field.
-		 */
-		menvcfg_val |= ENVCFG_CBCFE;
-
-		/*
-		 * Set menvcfg.CBIE == 3
-		 *
-		 * If Zicbom extension is not available then writes to
-		 * menvcfg.CBIE will be ignored because it is a WARL field.
-		 */
-		menvcfg_val |= ENVCFG_CBIE_INV << ENVCFG_CBIE_SHIFT;
-
-		/*
-		 * Set menvcfg.PBMTE == 1 for RV64 or RV128
-		 *
-		 * If Svpbmt extension is not available then menvcfg.PBMTE
-		 * will be read-only zero.
-		 */
-#if __riscv_xlen > 32
-		menvcfg_val |= ENVCFG_PBMTE;
-#endif
-
-		/*
-		 * The spec doesn't explicitly describe the reset value of menvcfg.
-		 * Enable access to stimecmp if sstc extension is present in the
-		 * hardware.
-		 */
-		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSTC)) {
 #if __riscv_xlen == 32
-			unsigned long menvcfgh_val;
-			menvcfgh_val = csr_read(CSR_MENVCFGH);
-			menvcfgh_val |= ENVCFGH_STCE;
-			csr_write(CSR_MENVCFGH, menvcfgh_val);
-#else
-			menvcfg_val |= ENVCFG_STCE;
+		menvcfg_val |= ((uint64_t)csr_read(CSR_MENVCFGH)) << 32;
 #endif
-		}
+
+#define __set_menvcfg_ext(__ext, __bits)				\
+		if (sbi_hart_has_extension(scratch, __ext))		\
+			menvcfg_val |= __bits;
+
+		/*
+		 * Enable access to extensions if they are present in the
+		 * hardware or in the device tree.
+		 */
+
+		__set_menvcfg_ext(SBI_HART_EXT_ZICBOZ, ENVCFG_CBZE)
+		__set_menvcfg_ext(SBI_HART_EXT_ZICBOM, ENVCFG_CBCFE)
+		__set_menvcfg_ext(SBI_HART_EXT_ZICBOM,
+				  ENVCFG_CBIE_INV << ENVCFG_CBIE_SHIFT)
+#if __riscv_xlen > 32
+		__set_menvcfg_ext(SBI_HART_EXT_SVPBMT, ENVCFG_PBMTE)
+#endif
+		__set_menvcfg_ext(SBI_HART_EXT_SSTC, ENVCFG_STCE)
+
+#undef __set_menvcfg_ext
 
 		csr_write(CSR_MENVCFG, menvcfg_val);
+#if __riscv_xlen == 32
+		csr_write(CSR_MENVCFGH, menvcfg_val >> 32);
+#endif
+
+		/* Enable S-mode access to seed CSR */
+		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_ZKR)) {
+			csr_set(CSR_MSECCFG, MSECCFG_SSEED);
+			csr_clear(CSR_MSECCFG, MSECCFG_USEED);
+		}
 	}
 
 	/* Disable all interrupts */
@@ -243,12 +227,12 @@ void sbi_hart_delegation_dump(struct sbi_scratch *scratch,
 		   prefix, suffix, csr_read(CSR_MEDELEG));
 }
 
-unsigned int sbi_hart_mhpm_count(struct sbi_scratch *scratch)
+unsigned int sbi_hart_mhpm_mask(struct sbi_scratch *scratch)
 {
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 
-	return hfeatures->mhpm_count;
+	return hfeatures->mhpm_mask;
 }
 
 unsigned int sbi_hart_pmp_count(struct sbi_scratch *scratch)
@@ -259,12 +243,12 @@ unsigned int sbi_hart_pmp_count(struct sbi_scratch *scratch)
 	return hfeatures->pmp_count;
 }
 
-unsigned long sbi_hart_pmp_granularity(struct sbi_scratch *scratch)
+unsigned int sbi_hart_pmp_log2gran(struct sbi_scratch *scratch)
 {
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 
-	return hfeatures->pmp_gran;
+	return hfeatures->pmp_log2gran;
 }
 
 unsigned int sbi_hart_pmp_addrbits(struct sbi_scratch *scratch)
@@ -283,45 +267,273 @@ unsigned int sbi_hart_mhpm_bits(struct sbi_scratch *scratch)
 	return hfeatures->mhpm_bits;
 }
 
-int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
+/*
+ * Returns Smepmp flags for a given domain and region based on permissions.
+ */
+static unsigned int sbi_hart_get_smepmp_flags(struct sbi_scratch *scratch,
+					      struct sbi_domain *dom,
+					      struct sbi_domain_memregion *reg)
+{
+	unsigned int pmp_flags = 0;
+
+	if (SBI_DOMAIN_MEMREGION_IS_SHARED(reg->flags)) {
+		/* Read only for both M and SU modes */
+		if (SBI_DOMAIN_MEMREGION_IS_SUR_MR(reg->flags))
+			pmp_flags = (PMP_L | PMP_R | PMP_W | PMP_X);
+
+		/* Execute for SU but Read/Execute for M mode */
+		else if (SBI_DOMAIN_MEMREGION_IS_SUX_MRX(reg->flags))
+			/* locked region */
+			pmp_flags = (PMP_L | PMP_W | PMP_X);
+
+		/* Execute only for both M and SU modes */
+		else if (SBI_DOMAIN_MEMREGION_IS_SUX_MX(reg->flags))
+			pmp_flags = (PMP_L | PMP_W);
+
+		/* Read/Write for both M and SU modes */
+		else if (SBI_DOMAIN_MEMREGION_IS_SURW_MRW(reg->flags))
+			pmp_flags = (PMP_W | PMP_X);
+
+		/* Read only for SU mode but Read/Write for M mode */
+		else if (SBI_DOMAIN_MEMREGION_IS_SUR_MRW(reg->flags))
+			pmp_flags = (PMP_W);
+	} else if (SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(reg->flags)) {
+		/*
+		 * When smepmp is supported and used, M region cannot have RWX
+		 * permissions on any region.
+		 */
+		if ((reg->flags & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK)
+		    == SBI_DOMAIN_MEMREGION_M_RWX) {
+			sbi_printf("%s: M-mode only regions cannot have"
+				   "RWX permissions\n", __func__);
+			return 0;
+		}
+
+		/* M-mode only access regions are always locked */
+		pmp_flags |= PMP_L;
+
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_READABLE)
+			pmp_flags |= PMP_R;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_WRITABLE)
+			pmp_flags |= PMP_W;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_EXECUTABLE)
+			pmp_flags |= PMP_X;
+	} else if (SBI_DOMAIN_MEMREGION_SU_ONLY_ACCESS(reg->flags)) {
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_READABLE)
+			pmp_flags |= PMP_R;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE)
+			pmp_flags |= PMP_W;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+			pmp_flags |= PMP_X;
+	}
+
+	return pmp_flags;
+}
+
+static void sbi_hart_smepmp_set(struct sbi_scratch *scratch,
+				struct sbi_domain *dom,
+				struct sbi_domain_memregion *reg,
+				unsigned int pmp_idx,
+				unsigned int pmp_flags,
+				unsigned int pmp_log2gran,
+				unsigned long pmp_addr_max)
+{
+	unsigned long pmp_addr = reg->base >> PMP_SHIFT;
+
+	if (pmp_log2gran <= reg->order && pmp_addr < pmp_addr_max) {
+		pmp_set(pmp_idx, pmp_flags, reg->base, reg->order);
+	} else {
+		sbi_printf("Can not configure pmp for domain %s because"
+			   " memory region address 0x%lx or size 0x%lx "
+			   "is not in range.\n", dom->name, reg->base,
+			   reg->order);
+	}
+}
+
+static int sbi_hart_smepmp_configure(struct sbi_scratch *scratch,
+				     unsigned int pmp_count,
+				     unsigned int pmp_log2gran,
+				     unsigned long pmp_addr_max)
 {
 	struct sbi_domain_memregion *reg;
 	struct sbi_domain *dom = sbi_domain_thishart_ptr();
-	unsigned int pmp_idx = 0, pmp_flags, pmp_bits, pmp_gran_log2;
-	unsigned int pmp_count = sbi_hart_pmp_count(scratch);
-	unsigned long pmp_addr = 0, pmp_addr_max = 0;
+	unsigned int pmp_idx, pmp_flags;
 
-	if (!pmp_count)
-		return 0;
+	/*
+	 * Set the RLB so that, we can write to PMP entries without
+	 * enforcement even if some entries are locked.
+	 */
+	csr_set(CSR_MSECCFG, MSECCFG_RLB);
 
-	pmp_gran_log2 = log2roundup(sbi_hart_pmp_granularity(scratch));
-	pmp_bits = sbi_hart_pmp_addrbits(scratch) - 1;
-	pmp_addr_max = (1UL << pmp_bits) | ((1UL << pmp_bits) - 1);
+	/* Disable the reserved entry */
+	pmp_disable(SBI_SMEPMP_RESV_ENTRY);
+
+	/* Program M-only regions when MML is not set. */
+	pmp_idx = 0;
+	sbi_domain_for_each_memregion(dom, reg) {
+		/* Skip reserved entry */
+		if (pmp_idx == SBI_SMEPMP_RESV_ENTRY)
+			pmp_idx++;
+		if (pmp_count <= pmp_idx)
+			break;
+
+		/* Skip shared and SU-only regions */
+		if (!SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(reg->flags)) {
+			pmp_idx++;
+			continue;
+		}
+
+		pmp_flags = sbi_hart_get_smepmp_flags(scratch, dom, reg);
+		if (!pmp_flags)
+			return 0;
+
+		sbi_hart_smepmp_set(scratch, dom, reg, pmp_idx++, pmp_flags,
+				    pmp_log2gran, pmp_addr_max);
+	}
+
+	/* Set the MML to enforce new encoding */
+	csr_set(CSR_MSECCFG, MSECCFG_MML);
+
+	/* Program shared and SU-only regions */
+	pmp_idx = 0;
+	sbi_domain_for_each_memregion(dom, reg) {
+		/* Skip reserved entry */
+		if (pmp_idx == SBI_SMEPMP_RESV_ENTRY)
+			pmp_idx++;
+		if (pmp_count <= pmp_idx)
+			break;
+
+		/* Skip M-only regions */
+		if (SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(reg->flags)) {
+			pmp_idx++;
+			continue;
+		}
+
+		pmp_flags = sbi_hart_get_smepmp_flags(scratch, dom, reg);
+		if (!pmp_flags)
+			return 0;
+
+		sbi_hart_smepmp_set(scratch, dom, reg, pmp_idx++, pmp_flags,
+				    pmp_log2gran, pmp_addr_max);
+	}
+
+	/*
+	 * All entries are programmed.
+	 * Keep the RLB bit so that dynamic mappings can be done.
+	 */
+
+	return 0;
+}
+
+static int sbi_hart_oldpmp_configure(struct sbi_scratch *scratch,
+				     unsigned int pmp_count,
+				     unsigned int pmp_log2gran,
+				     unsigned long pmp_addr_max)
+{
+	struct sbi_domain_memregion *reg;
+	struct sbi_domain *dom = sbi_domain_thishart_ptr();
+	unsigned int pmp_idx = 0;
+	unsigned int pmp_flags;
+	unsigned long pmp_addr;
 
 	sbi_domain_for_each_memregion(dom, reg) {
 		if (pmp_count <= pmp_idx)
 			break;
 
 		pmp_flags = 0;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_READABLE)
-			pmp_flags |= PMP_R;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_WRITEABLE)
-			pmp_flags |= PMP_W;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_EXECUTABLE)
-			pmp_flags |= PMP_X;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_MMODE)
+
+		/*
+		 * If permissions are to be enforced for all modes on
+		 * this region, the lock bit should be set.
+		 */
+		if (reg->flags & SBI_DOMAIN_MEMREGION_ENF_PERMISSIONS)
 			pmp_flags |= PMP_L;
 
-		pmp_addr =  reg->base >> PMP_SHIFT;
-		if (pmp_gran_log2 <= reg->order && pmp_addr < pmp_addr_max)
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_READABLE)
+			pmp_flags |= PMP_R;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE)
+			pmp_flags |= PMP_W;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+			pmp_flags |= PMP_X;
+
+		pmp_addr = reg->base >> PMP_SHIFT;
+		if (pmp_log2gran <= reg->order && pmp_addr < pmp_addr_max) {
 			pmp_set(pmp_idx++, pmp_flags, reg->base, reg->order);
-		else {
-			sbi_printf("Can not configure pmp for domain %s", dom->name);
-			sbi_printf(" because memory region address %lx or size %lx is not in range\n",
-				    reg->base, reg->order);
+		} else {
+			sbi_printf("Can not configure pmp for domain %s because"
+				   " memory region address 0x%lx or size 0x%lx "
+				   "is not in range.\n", dom->name, reg->base,
+				   reg->order);
 		}
 	}
 
+	return 0;
+}
+
+int sbi_hart_map_saddr(unsigned long addr, unsigned long size)
+{
+	/* shared R/W access for M and S/U mode */
+	unsigned int pmp_flags = (PMP_W | PMP_X);
+	unsigned long order, base = 0;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	/* If Smepmp is not supported no special mapping is required */
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP))
+		return SBI_OK;
+
+	if (is_pmp_entry_mapped(SBI_SMEPMP_RESV_ENTRY))
+		return SBI_ENOSPC;
+
+	for (order = MAX(sbi_hart_pmp_log2gran(scratch), log2roundup(size));
+	     order <= __riscv_xlen; order++) {
+		if (order < __riscv_xlen) {
+			base = addr & ~((1UL << order) - 1UL);
+			if ((base <= addr) &&
+			    (addr < (base + (1UL << order))) &&
+			    (base <= (addr + size - 1UL)) &&
+			    ((addr + size - 1UL) < (base + (1UL << order))))
+				break;
+		} else {
+			return SBI_EFAIL;
+		}
+	}
+
+	pmp_set(SBI_SMEPMP_RESV_ENTRY, pmp_flags, base, order);
+
+	return SBI_OK;
+}
+
+int sbi_hart_unmap_saddr(void)
+{
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP))
+		return SBI_OK;
+
+	return pmp_disable(SBI_SMEPMP_RESV_ENTRY);
+}
+
+int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
+{
+	int rc;
+	unsigned int pmp_bits, pmp_log2gran;
+	unsigned int pmp_count = sbi_hart_pmp_count(scratch);
+	unsigned long pmp_addr_max;
+
+	if (!pmp_count)
+		return 0;
+
+	pmp_log2gran = sbi_hart_pmp_log2gran(scratch);
+	pmp_bits = sbi_hart_pmp_addrbits(scratch) - 1;
+	pmp_addr_max = (1UL << pmp_bits) | ((1UL << pmp_bits) - 1);
+
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP))
+		rc = sbi_hart_smepmp_configure(scratch, pmp_count,
+						pmp_log2gran, pmp_addr_max);
+	else
+		rc = sbi_hart_oldpmp_configure(scratch, pmp_count,
+						pmp_log2gran, pmp_addr_max);
+
 	/*
 	 * As per section 3.7.2 of privileged specification v1.12,
 	 * virtual address translations can be speculatively performed
@@ -343,7 +555,7 @@ int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
 			__sbi_hfence_gvma_all();
 	}
 
-	return 0;
+	return rc;
 }
 
 int sbi_hart_priv_version(struct sbi_scratch *scratch)
@@ -385,9 +597,9 @@ static inline void __sbi_hart_update_extension(
 					bool enable)
 {
 	if (enable)
-		hfeatures->extensions |= BIT(ext);
+		__set_bit(ext, hfeatures->extensions);
 	else
-		hfeatures->extensions &= ~BIT(ext);
+		__clear_bit(ext, hfeatures->extensions);
 }
 
 /**
@@ -420,39 +632,33 @@ bool sbi_hart_has_extension(struct sbi_scratch *scratch,
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 
-	if (hfeatures->extensions & BIT(ext))
+	if (__test_bit(ext, hfeatures->extensions))
 		return true;
 	else
 		return false;
 }
 
-static inline char *sbi_hart_extension_id2string(int ext)
-{
-	char *estr = NULL;
-
-	switch (ext) {
-	case SBI_HART_EXT_SSCOFPMF:
-		estr = "sscofpmf";
-		break;
-	case SBI_HART_EXT_TIME:
-		estr = "time";
-		break;
-	case SBI_HART_EXT_SMAIA:
-		estr = "smaia";
-		break;
-	case SBI_HART_EXT_SSTC:
-		estr = "sstc";
-		break;
-	case SBI_HART_EXT_SMSTATEEN:
-		estr = "smstateen";
-		break;
-	default:
-		break;
-	}
-
-	return estr;
+#define __SBI_HART_EXT_DATA(_name, _id) {	\
+	.name = #_name,				\
+	.id = _id,				\
 }
 
+const struct sbi_hart_ext_data sbi_hart_ext[] = {
+	__SBI_HART_EXT_DATA(smaia, SBI_HART_EXT_SMAIA),
+	__SBI_HART_EXT_DATA(smepmp, SBI_HART_EXT_SMEPMP),
+	__SBI_HART_EXT_DATA(smstateen, SBI_HART_EXT_SMSTATEEN),
+	__SBI_HART_EXT_DATA(sscofpmf, SBI_HART_EXT_SSCOFPMF),
+	__SBI_HART_EXT_DATA(sstc, SBI_HART_EXT_SSTC),
+	__SBI_HART_EXT_DATA(zicntr, SBI_HART_EXT_ZICNTR),
+	__SBI_HART_EXT_DATA(zihpm, SBI_HART_EXT_ZIHPM),
+	__SBI_HART_EXT_DATA(zkr, SBI_HART_EXT_ZKR),
+	__SBI_HART_EXT_DATA(smcntrpmf, SBI_HART_EXT_SMCNTRPMF),
+	__SBI_HART_EXT_DATA(xandespmu, SBI_HART_EXT_XANDESPMU),
+	__SBI_HART_EXT_DATA(zicboz, SBI_HART_EXT_ZICBOZ),
+	__SBI_HART_EXT_DATA(zicbom, SBI_HART_EXT_ZICBOM),
+	__SBI_HART_EXT_DATA(svpbmt, SBI_HART_EXT_SVPBMT),
+};
+
 /**
  * Get the hart extensions in string format
  *
@@ -468,30 +674,18 @@ void sbi_hart_get_extensions_str(struct sbi_scratch *scratch,
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 	int offset = 0, ext = 0;
-	char *temp;
 
 	if (!extensions_str || nestr <= 0)
 		return;
 	sbi_memset(extensions_str, 0, nestr);
 
-	if (!hfeatures->extensions)
-		goto done;
+	for_each_set_bit(ext, hfeatures->extensions, SBI_HART_EXT_MAX) {
+		sbi_snprintf(extensions_str + offset,
+				 nestr - offset,
+				 "%s,", sbi_hart_ext[ext].name);
+		offset = offset + sbi_strlen(sbi_hart_ext[ext].name) + 1;
+	}
 
-	do {
-		if (hfeatures->extensions & BIT(ext)) {
-			temp = sbi_hart_extension_id2string(ext);
-			if (temp) {
-				sbi_snprintf(extensions_str + offset,
-					     nestr - offset,
-					     "%s,", temp);
-				offset = offset + sbi_strlen(temp) + 1;
-			}
-		}
-
-		ext++;
-	} while (ext < SBI_HART_EXT_MAX);
-
-done:
 	if (offset)
 		extensions_str[offset - 1] = '\0';
 	else
@@ -517,7 +711,7 @@ static unsigned long hart_pmp_get_allowed_addr(void)
 	return val;
 }
 
-static int hart_pmu_get_allowed_bits(void)
+static int hart_mhpm_get_allowed_bits(void)
 {
 	unsigned long val = ~(0UL);
 	struct sbi_trap_info trap = {0};
@@ -554,6 +748,7 @@ static int hart_detect_features(struct sbi_scratch *scratch)
 	struct sbi_hart_features *hfeatures =
 		sbi_scratch_offset_ptr(scratch, hart_features_offset);
 	unsigned long val, oldval;
+	bool has_zicntr = false;
 	int rc;
 
 	/* If hart features already detected then do nothing */
@@ -561,9 +756,32 @@ static int hart_detect_features(struct sbi_scratch *scratch)
 		return 0;
 
 	/* Clear hart features */
-	hfeatures->extensions = 0;
+	sbi_memset(hfeatures->extensions, 0, sizeof(hfeatures->extensions));
 	hfeatures->pmp_count = 0;
-	hfeatures->mhpm_count = 0;
+	hfeatures->mhpm_mask = 0;
+	hfeatures->priv_version = SBI_HART_PRIV_VER_UNKNOWN;
+
+#define __check_hpm_csr(__csr, __mask) 					  \
+	oldval = csr_read_allowed(__csr, (ulong)&trap);			  \
+	if (!trap.cause) {						  \
+		csr_write_allowed(__csr, (ulong)&trap, 1UL);		  \
+		if (!trap.cause && csr_swap(__csr, oldval) == 1UL) {	  \
+			(hfeatures->__mask) |= 1 << (__csr - CSR_MCYCLE); \
+		}							  \
+	}
+
+#define __check_hpm_csr_2(__csr, __mask)	 		  \
+	__check_hpm_csr(__csr + 0, __mask)	 		  \
+	__check_hpm_csr(__csr + 1, __mask)
+#define __check_hpm_csr_4(__csr, __mask)	 		  \
+	__check_hpm_csr_2(__csr + 0, __mask) 			  \
+	__check_hpm_csr_2(__csr + 2, __mask)
+#define __check_hpm_csr_8(__csr, __mask)	 		  \
+	__check_hpm_csr_4(__csr + 0, __mask) 			  \
+	__check_hpm_csr_4(__csr + 4, __mask)
+#define __check_hpm_csr_16(__csr, __mask)	 		  \
+	__check_hpm_csr_8(__csr + 0, __mask) 			  \
+	__check_hpm_csr_8(__csr + 8, __mask)
 
 #define __check_csr(__csr, __rdonly, __wrval, __field, __skip)	\
 	oldval = csr_read_allowed(__csr, (ulong)&trap);			\
@@ -609,28 +827,23 @@ static int hart_detect_features(struct sbi_scratch *scratch)
 	 */
 	val = hart_pmp_get_allowed_addr();
 	if (val) {
-		hfeatures->pmp_gran =  1 << (sbi_ffs(val) + 2);
+		hfeatures->pmp_log2gran = sbi_ffs(val) + 2;
 		hfeatures->pmp_addr_bits = sbi_fls(val) + 1;
 		/* Detect number of PMP regions. At least PMPADDR0 should be implemented*/
 		__check_csr_64(CSR_PMPADDR0, 0, val, pmp_count, __pmp_skip);
 	}
 __pmp_skip:
-
 	/* Detect number of MHPM counters */
-	__check_csr(CSR_MHPMCOUNTER3, 0, 1UL, mhpm_count, __mhpm_skip);
-	hfeatures->mhpm_bits = hart_pmu_get_allowed_bits();
-
-	__check_csr_4(CSR_MHPMCOUNTER4, 0, 1UL, mhpm_count, __mhpm_skip);
-	__check_csr_8(CSR_MHPMCOUNTER8, 0, 1UL, mhpm_count, __mhpm_skip);
-	__check_csr_16(CSR_MHPMCOUNTER16, 0, 1UL, mhpm_count, __mhpm_skip);
+	__check_hpm_csr(CSR_MHPMCOUNTER3, mhpm_mask);
+	hfeatures->mhpm_bits = hart_mhpm_get_allowed_bits();
+	__check_hpm_csr_4(CSR_MHPMCOUNTER4, mhpm_mask);
+	__check_hpm_csr_8(CSR_MHPMCOUNTER8, mhpm_mask);
+	__check_hpm_csr_16(CSR_MHPMCOUNTER16, mhpm_mask);
 
 	/**
 	 * No need to check for MHPMCOUNTERH for RV32 as they are expected to be
 	 * implemented if MHPMCOUNTER is implemented.
 	 */
-
-__mhpm_skip:
-
 #undef __check_csr_64
 #undef __check_csr_32
 #undef __check_csr_16
@@ -639,59 +852,57 @@ __mhpm_skip:
 #undef __check_csr_2
 #undef __check_csr
 
+
+#define __check_priv(__csr, __base_priv, __priv)			\
+	val = csr_read_allowed(__csr, (ulong)&trap);			\
+	if (!trap.cause && (hfeatures->priv_version >= __base_priv)) {	\
+		hfeatures->priv_version = __priv;			\
+	}
+
 	/* Detect if hart supports Priv v1.10 */
-	val = csr_read_allowed(CSR_MCOUNTEREN, (unsigned long)&trap);
-	if (!trap.cause)
-		hfeatures->priv_version = SBI_HART_PRIV_VER_1_10;
-
+	__check_priv(CSR_MCOUNTEREN,
+		     SBI_HART_PRIV_VER_UNKNOWN, SBI_HART_PRIV_VER_1_10);
 	/* Detect if hart supports Priv v1.11 */
-	val = csr_read_allowed(CSR_MCOUNTINHIBIT, (unsigned long)&trap);
-	if (!trap.cause &&
-	    (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_10))
-		hfeatures->priv_version = SBI_HART_PRIV_VER_1_11;
-
+	__check_priv(CSR_MCOUNTINHIBIT,
+		     SBI_HART_PRIV_VER_1_10, SBI_HART_PRIV_VER_1_11);
 	/* Detect if hart supports Priv v1.12 */
-	csr_read_allowed(CSR_MENVCFG, (unsigned long)&trap);
-	if (!trap.cause &&
-	    (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_11))
-		hfeatures->priv_version = SBI_HART_PRIV_VER_1_12;
+	__check_priv(CSR_MENVCFG,
+		     SBI_HART_PRIV_VER_1_11, SBI_HART_PRIV_VER_1_12);
+
+#undef __check_priv_csr
+
+#define __check_ext_csr(__base_priv, __csr, __ext)			\
+	if (hfeatures->priv_version >= __base_priv) {			\
+		csr_read_allowed(__csr, (ulong)&trap);			\
+		if (!trap.cause)					\
+			__sbi_hart_update_extension(hfeatures,		\
+						    __ext, true);	\
+	}
 
 	/* Counter overflow/filtering is not useful without mcounter/inhibit */
-	if (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_12) {
-		/* Detect if hart supports sscofpmf */
-		csr_read_allowed(CSR_SCOUNTOVF, (unsigned long)&trap);
-		if (!trap.cause)
-			__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SSCOFPMF, true);
-	}
-
+	/* Detect if hart supports sscofpmf */
+	__check_ext_csr(SBI_HART_PRIV_VER_1_11,
+			CSR_SCOUNTOVF, SBI_HART_EXT_SSCOFPMF);
 	/* Detect if hart supports time CSR */
-	csr_read_allowed(CSR_TIME, (unsigned long)&trap);
-	if (!trap.cause)
-		__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_TIME, true);
-
+	__check_ext_csr(SBI_HART_PRIV_VER_UNKNOWN,
+			CSR_TIME, SBI_HART_EXT_ZICNTR);
 	/* Detect if hart has AIA local interrupt CSRs */
-	csr_read_allowed(CSR_MTOPI, (unsigned long)&trap);
-	if (!trap.cause)
-		__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SMAIA, true);
-
+	__check_ext_csr(SBI_HART_PRIV_VER_UNKNOWN,
+			CSR_MTOPI, SBI_HART_EXT_SMAIA);
 	/* Detect if hart supports stimecmp CSR(Sstc extension) */
-	if (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_12) {
-		csr_read_allowed(CSR_STIMECMP, (unsigned long)&trap);
-		if (!trap.cause)
-			__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SSTC, true);
-	}
-
+	__check_ext_csr(SBI_HART_PRIV_VER_1_12,
+			CSR_STIMECMP, SBI_HART_EXT_SSTC);
 	/* Detect if hart supports mstateen CSRs */
-	if (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_12) {
-		val = csr_read_allowed(CSR_MSTATEEN0, (unsigned long)&trap);
-		if (!trap.cause)
-			__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SMSTATEEN, true);
-	}
+	__check_ext_csr(SBI_HART_PRIV_VER_1_12,
+			CSR_MSTATEEN0, SBI_HART_EXT_SMSTATEEN);
+	/* Detect if hart supports smcntrpmf */
+	__check_ext_csr(SBI_HART_PRIV_VER_1_12,
+			CSR_MCYCLECFG, SBI_HART_EXT_SMCNTRPMF);
+
+#undef __check_ext_csr
+
+	/* Save trap based detection of Zicntr */
+	has_zicntr = sbi_hart_has_extension(scratch, SBI_HART_EXT_ZICNTR);
 
 	/* Let platform populate extensions */
 	rc = sbi_platform_extensions_init(sbi_platform_thishart_ptr(),
@@ -699,9 +910,28 @@ __mhpm_skip:
 	if (rc)
 		return rc;
 
+	/* Zicntr should only be detected using traps */
+	__sbi_hart_update_extension(hfeatures, SBI_HART_EXT_ZICNTR,
+				    has_zicntr);
+
+	/* Extensions implied by other extensions and features */
+	if (hfeatures->mhpm_mask)
+		__sbi_hart_update_extension(hfeatures,
+					SBI_HART_EXT_ZIHPM, true);
+
 	/* Mark hart feature detection done */
 	hfeatures->detected = true;
 
+	/*
+	 * On platforms with Smepmp, the previous booting stage must
+	 * enter OpenSBI with mseccfg.MML == 0. This allows OpenSBI
+	 * to configure it's own M-mode only regions without depending
+	 * on the previous booting stage.
+	 */
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP) &&
+	    (csr_read(CSR_MSECCFG) & MSECCFG_MML))
+		return SBI_EILL;
+
 	return 0;
 }
 
@@ -726,6 +956,12 @@ int sbi_hart_init(struct sbi_scratch *scratch, bool cold_boot)
 {
 	int rc;
 
+	/*
+	 * Clear mip CSR before proceeding with init to avoid any spurious
+	 * external interrupts in S-mode.
+	 */
+	csr_write(CSR_MIP, 0);
+
 	if (cold_boot) {
 		if (misa_extension('H'))
 			sbi_hart_expected_trap = &__sbi_expected_trap_hext;

lib/sbi/sbi_heap.c

@@ -0,0 +1,204 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel<apatel@ventanamicro.com>
+ */
+
+#include <sbi/riscv_locks.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_heap.h>
+#include <sbi/sbi_list.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+
+/* Minimum size and alignment of heap allocations */
+#define HEAP_ALLOC_ALIGN		64
+#define HEAP_HOUSEKEEPING_FACTOR	16
+
+struct heap_node {
+	struct sbi_dlist head;
+	unsigned long addr;
+	unsigned long size;
+};
+
+struct heap_control {
+	spinlock_t lock;
+	unsigned long base;
+	unsigned long size;
+	unsigned long hkbase;
+	unsigned long hksize;
+	struct sbi_dlist free_node_list;
+	struct sbi_dlist free_space_list;
+	struct sbi_dlist used_space_list;
+};
+
+static struct heap_control hpctrl;
+
+void *sbi_malloc(size_t size)
+{
+	void *ret = NULL;
+	struct heap_node *n, *np;
+
+	if (!size)
+		return NULL;
+
+	size += HEAP_ALLOC_ALIGN - 1;
+	size &= ~((unsigned long)HEAP_ALLOC_ALIGN - 1);
+
+	spin_lock(&hpctrl.lock);
+
+	np = NULL;
+	sbi_list_for_each_entry(n, &hpctrl.free_space_list, head) {
+		if (size <= n->size) {
+			np = n;
+			break;
+		}
+	}
+	if (np) {
+		if ((size < np->size) &&
+		    !sbi_list_empty(&hpctrl.free_node_list)) {
+			n = sbi_list_first_entry(&hpctrl.free_node_list,
+						 struct heap_node, head);
+			sbi_list_del(&n->head);
+			n->addr = np->addr + np->size - size;
+			n->size = size;
+			np->size -= size;
+			sbi_list_add_tail(&n->head, &hpctrl.used_space_list);
+			ret = (void *)n->addr;
+		} else if (size == np->size) {
+			sbi_list_del(&np->head);
+			sbi_list_add_tail(&np->head, &hpctrl.used_space_list);
+			ret = (void *)np->addr;
+		}
+	}
+
+	spin_unlock(&hpctrl.lock);
+
+	return ret;
+}
+
+void *sbi_zalloc(size_t size)
+{
+	void *ret = sbi_malloc(size);
+
+	if (ret)
+		sbi_memset(ret, 0, size);
+	return ret;
+}
+
+void sbi_free(void *ptr)
+{
+	struct heap_node *n, *np;
+
+	if (!ptr)
+		return;
+
+	spin_lock(&hpctrl.lock);
+
+	np = NULL;
+	sbi_list_for_each_entry(n, &hpctrl.used_space_list, head) {
+		if ((n->addr <= (unsigned long)ptr) &&
+		    ((unsigned long)ptr < (n->addr + n->size))) {
+			np = n;
+			break;
+		}
+	}
+	if (!np) {
+		spin_unlock(&hpctrl.lock);
+		return;
+	}
+
+	sbi_list_del(&np->head);
+
+	sbi_list_for_each_entry(n, &hpctrl.free_space_list, head) {
+		if ((np->addr + np->size) == n->addr) {
+			n->addr = np->addr;
+			n->size += np->size;
+			sbi_list_add_tail(&np->head, &hpctrl.free_node_list);
+			np = NULL;
+			break;
+		} else if (np->addr == (n->addr + n->size)) {
+			n->size += np->size;
+			sbi_list_add_tail(&np->head, &hpctrl.free_node_list);
+			np = NULL;
+			break;
+		} else if ((n->addr + n->size) < np->addr) {
+			sbi_list_add(&np->head, &n->head);
+			np = NULL;
+			break;
+		}
+	}
+	if (np)
+		sbi_list_add_tail(&np->head, &hpctrl.free_space_list);
+
+	spin_unlock(&hpctrl.lock);
+}
+
+unsigned long sbi_heap_free_space(void)
+{
+	struct heap_node *n;
+	unsigned long ret = 0;
+
+	spin_lock(&hpctrl.lock);
+	sbi_list_for_each_entry(n, &hpctrl.free_space_list, head)
+		ret += n->size;
+	spin_unlock(&hpctrl.lock);
+
+	return ret;
+}
+
+unsigned long sbi_heap_used_space(void)
+{
+	return hpctrl.size - hpctrl.hksize - sbi_heap_free_space();
+}
+
+unsigned long sbi_heap_reserved_space(void)
+{
+	return hpctrl.hksize;
+}
+
+int sbi_heap_init(struct sbi_scratch *scratch)
+{
+	unsigned long i;
+	struct heap_node *n;
+
+	/* Sanity checks on heap offset and size */
+	if (!scratch->fw_heap_size ||
+	    (scratch->fw_heap_size & (HEAP_BASE_ALIGN - 1)) ||
+	    (scratch->fw_heap_offset < scratch->fw_rw_offset) ||
+	    (scratch->fw_size < (scratch->fw_heap_offset + scratch->fw_heap_size)) ||
+	    (scratch->fw_heap_offset & (HEAP_BASE_ALIGN - 1)))
+		return SBI_EINVAL;
+
+	/* Initialize heap control */
+	SPIN_LOCK_INIT(hpctrl.lock);
+	hpctrl.base = scratch->fw_start + scratch->fw_heap_offset;
+	hpctrl.size = scratch->fw_heap_size;
+	hpctrl.hkbase = hpctrl.base;
+	hpctrl.hksize = hpctrl.size / HEAP_HOUSEKEEPING_FACTOR;
+	hpctrl.hksize &= ~((unsigned long)HEAP_BASE_ALIGN - 1);
+	SBI_INIT_LIST_HEAD(&hpctrl.free_node_list);
+	SBI_INIT_LIST_HEAD(&hpctrl.free_space_list);
+	SBI_INIT_LIST_HEAD(&hpctrl.used_space_list);
+
+	/* Prepare free node list */
+	for (i = 0; i < (hpctrl.hksize / sizeof(*n)); i++) {
+		n = (struct heap_node *)(hpctrl.hkbase + (sizeof(*n) * i));
+		SBI_INIT_LIST_HEAD(&n->head);
+		n->addr = n->size = 0;
+		sbi_list_add_tail(&n->head, &hpctrl.free_node_list);
+	}
+
+	/* Prepare free space list */
+	n = sbi_list_first_entry(&hpctrl.free_node_list,
+				 struct heap_node, head);
+	sbi_list_del(&n->head);
+	n->addr = hpctrl.hkbase + hpctrl.hksize;
+	n->size = hpctrl.size - hpctrl.hksize;
+	sbi_list_add_tail(&n->head, &hpctrl.free_space_list);
+
+	return 0;
+}

lib/sbi/sbi_hsm.c

@@ -26,6 +26,15 @@
 #include <sbi/sbi_timer.h>
 #include <sbi/sbi_console.h>
 
+#define __sbi_hsm_hart_change_state(hdata, oldstate, newstate)		\
+({									\
+	long state = atomic_cmpxchg(&(hdata)->state, oldstate, newstate); \
+	if (state != (oldstate))					\
+		sbi_printf("%s: ERR: The hart is in invalid state [%lu]\n", \
+			   __func__, state);				\
+	state == (oldstate);						\
+})
+
 static const struct sbi_hsm_device *hsm_dev = NULL;
 static unsigned long hart_data_offset;
 
@@ -35,9 +44,18 @@ struct sbi_hsm_data {
 	unsigned long suspend_type;
 	unsigned long saved_mie;
 	unsigned long saved_mip;
+	atomic_t start_ticket;
 };
 
-static inline int __sbi_hsm_hart_get_state(u32 hartid)
+bool sbi_hsm_hart_change_state(struct sbi_scratch *scratch, long oldstate,
+			       long newstate)
+{
+	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
+							    hart_data_offset);
+	return __sbi_hsm_hart_change_state(hdata, oldstate, newstate);
+}
+
+int __sbi_hsm_hart_get_state(u32 hartid)
 {
 	struct sbi_hsm_data *hdata;
 	struct sbi_scratch *scratch;
@@ -58,6 +76,32 @@ int sbi_hsm_hart_get_state(const struct sbi_domain *dom, u32 hartid)
 	return __sbi_hsm_hart_get_state(hartid);
 }
 
+/*
+ * Try to acquire the ticket for the given target hart to make sure only
+ * one hart prepares the start of the target hart.
+ * Returns true if the ticket has been acquired, false otherwise.
+ *
+ * The function has "acquire" semantics: no memory operations following it
+ * in the current hart can be seen before it by other harts.
+ * atomic_cmpxchg() provides the memory barriers needed for that.
+ */
+static bool hsm_start_ticket_acquire(struct sbi_hsm_data *hdata)
+{
+	return (atomic_cmpxchg(&hdata->start_ticket, 0, 1) == 0);
+}
+
+/*
+ * Release the ticket for the given target hart.
+ *
+ * The function has "release" semantics: no memory operations preceding it
+ * in the current hart can be seen after it by other harts.
+ */
+static void hsm_start_ticket_release(struct sbi_hsm_data *hdata)
+{
+	RISCV_FENCE(rw, w);
+	atomic_write(&hdata->start_ticket, 0);
+}
+
 /**
  * Get ulong HART mask for given HART base ID
  * @param dom the domain to be used for output HART mask
@@ -71,38 +115,46 @@ int sbi_hsm_hart_interruptible_mask(const struct sbi_domain *dom,
 {
 	int hstate;
 	ulong i, hmask, dmask;
-	ulong hend = sbi_scratch_last_hartid() + 1;
 
 	*out_hmask = 0;
-	if (hend <= hbase)
+	if (!sbi_hartid_valid(hbase))
 		return SBI_EINVAL;
-	if (BITS_PER_LONG < (hend - hbase))
-		hend = hbase + BITS_PER_LONG;
 
 	dmask = sbi_domain_get_assigned_hartmask(dom, hbase);
-	for (i = hbase; i < hend; i++) {
-		hmask = 1UL << (i - hbase);
-		if (dmask & hmask) {
-			hstate = __sbi_hsm_hart_get_state(i);
-			if (hstate == SBI_HSM_STATE_STARTED ||
-			    hstate == SBI_HSM_STATE_SUSPENDED)
-				*out_hmask |= hmask;
-		}
+	for (i = 0; i < BITS_PER_LONG; i++) {
+		hmask = 1UL << i;
+		if (!(dmask & hmask))
+			continue;
+
+		hstate = __sbi_hsm_hart_get_state(hbase + i);
+		if (hstate == SBI_HSM_STATE_STARTED ||
+		    hstate == SBI_HSM_STATE_SUSPENDED ||
+		    hstate == SBI_HSM_STATE_RESUME_PENDING)
+			*out_hmask |= hmask;
 	}
 
 	return 0;
 }
 
-void sbi_hsm_prepare_next_jump(struct sbi_scratch *scratch, u32 hartid)
+void __noreturn sbi_hsm_hart_start_finish(struct sbi_scratch *scratch,
+					  u32 hartid)
 {
-	u32 oldstate;
+	unsigned long next_arg1;
+	unsigned long next_addr;
+	unsigned long next_mode;
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
 
-	oldstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_START_PENDING,
-				  SBI_HSM_STATE_STARTED);
-	if (oldstate != SBI_HSM_STATE_START_PENDING)
+	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_START_PENDING,
+					 SBI_HSM_STATE_STARTED))
 		sbi_hart_hang();
+
+	next_arg1 = scratch->next_arg1;
+	next_addr = scratch->next_addr;
+	next_mode = scratch->next_mode;
+	hsm_start_ticket_release(hdata);
+
+	sbi_hart_switch_mode(hartid, next_arg1, next_addr, next_mode, false);
 }
 
 static void sbi_hsm_hart_wait(struct sbi_scratch *scratch, u32 hartid)
@@ -116,10 +168,10 @@ static void sbi_hsm_hart_wait(struct sbi_scratch *scratch, u32 hartid)
 	/* Set MSIE and MEIE bits to receive IPI */
 	csr_set(CSR_MIE, MIP_MSIP | MIP_MEIP);
 
-	/* Wait for hart_add call*/
+	/* Wait for state transition requested by sbi_hsm_hart_start() */
 	while (atomic_read(&hdata->state) != SBI_HSM_STATE_START_PENDING) {
 		wfi();
-	};
+	}
 
 	/* Restore MIE CSR */
 	csr_write(CSR_MIE, saved_mie);
@@ -196,17 +248,18 @@ int sbi_hsm_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot)
 			return SBI_ENOMEM;
 
 		/* Initialize hart state data for every hart */
-		for (i = 0; i <= sbi_scratch_last_hartid(); i++) {
-			rscratch = sbi_hartid_to_scratch(i);
+		for (i = 0; i <= sbi_scratch_last_hartindex(); i++) {
+			rscratch = sbi_hartindex_to_scratch(i);
 			if (!rscratch)
 				continue;
 
 			hdata = sbi_scratch_offset_ptr(rscratch,
 						       hart_data_offset);
 			ATOMIC_INIT(&hdata->state,
-				    (i == hartid) ?
+				    (sbi_hartindex_to_hartid(i) == hartid) ?
 				    SBI_HSM_STATE_START_PENDING :
 				    SBI_HSM_STATE_STOPPED);
+			ATOMIC_INIT(&hdata->start_ticket, 0);
 		}
 	} else {
 		sbi_hsm_hart_wait(scratch, hartid);
@@ -217,20 +270,17 @@ int sbi_hsm_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot)
 
 void __noreturn sbi_hsm_exit(struct sbi_scratch *scratch)
 {
-	u32 hstate;
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
 	void (*jump_warmboot)(void) = (void (*)(void))scratch->warmboot_addr;
 
-	hstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_STOP_PENDING,
-				SBI_HSM_STATE_STOPPED);
-	if (hstate != SBI_HSM_STATE_STOP_PENDING)
+	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_STOP_PENDING,
+					 SBI_HSM_STATE_STOPPED))
 		goto fail_exit;
 
 	if (hsm_device_has_hart_hotplug()) {
-		hsm_device_hart_stop();
-		/* It should never reach here */
-		goto fail_exit;
+		if (hsm_device_hart_stop() != SBI_ENOTSUPP)
+			goto fail_exit;
 	}
 
 	/**
@@ -248,12 +298,13 @@ fail_exit:
 
 int sbi_hsm_hart_start(struct sbi_scratch *scratch,
 		       const struct sbi_domain *dom,
-		       u32 hartid, ulong saddr, ulong smode, ulong priv)
+		       u32 hartid, ulong saddr, ulong smode, ulong arg1)
 {
-	unsigned long init_count;
+	unsigned long init_count, entry_count;
 	unsigned int hstate;
 	struct sbi_scratch *rscratch;
 	struct sbi_hsm_data *hdata;
+	int rc;
 
 	/* For now, we only allow start mode to be S-mode or U-mode. */
 	if (smode != PRV_S && smode != PRV_U)
@@ -267,39 +318,55 @@ int sbi_hsm_hart_start(struct sbi_scratch *scratch,
 	rscratch = sbi_hartid_to_scratch(hartid);
 	if (!rscratch)
 		return SBI_EINVAL;
+
 	hdata = sbi_scratch_offset_ptr(rscratch, hart_data_offset);
+	if (!hsm_start_ticket_acquire(hdata))
+		return SBI_EINVAL;
+
+	init_count = sbi_init_count(hartid);
+	entry_count = sbi_entry_count(hartid);
+
+	rscratch->next_arg1 = arg1;
+	rscratch->next_addr = saddr;
+	rscratch->next_mode = smode;
+
+	/*
+	 * atomic_cmpxchg() is an implicit barrier. It makes sure that
+	 * other harts see reading of init_count and writing to *rscratch
+	 * before hdata->state is set to SBI_HSM_STATE_START_PENDING.
+	 */
 	hstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_STOPPED,
 				SBI_HSM_STATE_START_PENDING);
-	if (hstate == SBI_HSM_STATE_STARTED)
-		return SBI_EALREADY;
+	if (hstate == SBI_HSM_STATE_STARTED) {
+		rc = SBI_EALREADY;
+		goto err;
+	}
 
 	/**
 	 * if a hart is already transition to start or stop, another start call
 	 * is considered as invalid request.
 	 */
-	if (hstate != SBI_HSM_STATE_STOPPED)
-		return SBI_EINVAL;
-
-	init_count = sbi_init_count(hartid);
-	rscratch->next_arg1 = priv;
-	rscratch->next_addr = saddr;
-	rscratch->next_mode = smode;
-
-	if (hsm_device_has_hart_hotplug() ||
-	   (hsm_device_has_hart_secondary_boot() && !init_count)) {
-		return hsm_device_hart_start(hartid, scratch->warmboot_addr);
-	} else {
-		int rc = sbi_ipi_raw_send(hartid);
-		if (rc)
-		    return rc;
+	if (hstate != SBI_HSM_STATE_STOPPED) {
+		rc = SBI_EINVAL;
+		goto err;
 	}
 
-	return 0;
+	if ((hsm_device_has_hart_hotplug() && (entry_count == init_count)) ||
+	   (hsm_device_has_hart_secondary_boot() && !init_count)) {
+		rc = hsm_device_hart_start(hartid, scratch->warmboot_addr);
+	} else {
+		rc = sbi_ipi_raw_send(sbi_hartid_to_hartindex(hartid));
+	}
+
+	if (!rc)
+		return 0;
+err:
+	hsm_start_ticket_release(hdata);
+	return rc;
 }
 
 int sbi_hsm_hart_stop(struct sbi_scratch *scratch, bool exitnow)
 {
-	int oldstate;
 	const struct sbi_domain *dom = sbi_domain_thishart_ptr();
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
@@ -307,13 +374,9 @@ int sbi_hsm_hart_stop(struct sbi_scratch *scratch, bool exitnow)
 	if (!dom)
 		return SBI_EFAIL;
 
-	oldstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_STARTED,
-				  SBI_HSM_STATE_STOP_PENDING);
-	if (oldstate != SBI_HSM_STATE_STARTED) {
-		sbi_printf("%s: ERR: The hart is in invalid state [%u]\n",
-			   __func__, oldstate);
+	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_STARTED,
+					 SBI_HSM_STATE_STOP_PENDING))
 		return SBI_EFAIL;
-	}
 
 	if (exitnow)
 		sbi_exit(scratch);
@@ -329,7 +392,7 @@ static int __sbi_hsm_suspend_default(struct sbi_scratch *scratch)
 	return 0;
 }
 
-static void __sbi_hsm_suspend_non_ret_save(struct sbi_scratch *scratch)
+void __sbi_hsm_suspend_non_ret_save(struct sbi_scratch *scratch)
 {
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
@@ -358,62 +421,55 @@ static void __sbi_hsm_suspend_non_ret_restore(struct sbi_scratch *scratch)
 							    hart_data_offset);
 
 	csr_write(CSR_MIE, hdata->saved_mie);
-	csr_write(CSR_MIP, (hdata->saved_mip & (MIP_SSIP | MIP_STIP)));
+	csr_set(CSR_MIP, (hdata->saved_mip & (MIP_SSIP | MIP_STIP)));
 }
 
 void sbi_hsm_hart_resume_start(struct sbi_scratch *scratch)
 {
-	int oldstate;
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
 
 	/* If current HART was SUSPENDED then set RESUME_PENDING state */
-	oldstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_SUSPENDED,
-			SBI_HSM_STATE_RESUME_PENDING);
-	if (oldstate != SBI_HSM_STATE_SUSPENDED) {
-		sbi_printf("%s: ERR: The hart is in invalid state [%u]\n",
-			   __func__, oldstate);
+	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_SUSPENDED,
+					 SBI_HSM_STATE_RESUME_PENDING))
 		sbi_hart_hang();
-	}
 
 	hsm_device_hart_resume();
 }
 
-void sbi_hsm_hart_resume_finish(struct sbi_scratch *scratch)
+void __noreturn sbi_hsm_hart_resume_finish(struct sbi_scratch *scratch,
+					   u32 hartid)
 {
-	u32 oldstate;
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
 
 	/* If current HART was RESUME_PENDING then set STARTED state */
-	oldstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_RESUME_PENDING,
-				  SBI_HSM_STATE_STARTED);
-	if (oldstate != SBI_HSM_STATE_RESUME_PENDING) {
-		sbi_printf("%s: ERR: The hart is in invalid state [%u]\n",
-			   __func__, oldstate);
+	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_RESUME_PENDING,
+					 SBI_HSM_STATE_STARTED))
 		sbi_hart_hang();
-	}
 
 	/*
 	 * Restore some of the M-mode CSRs which we are re-configured by
 	 * the warm-boot sequence.
 	 */
 	__sbi_hsm_suspend_non_ret_restore(scratch);
+
+	sbi_hart_switch_mode(hartid, scratch->next_arg1,
+			     scratch->next_addr,
+			     scratch->next_mode, false);
 }
 
 int sbi_hsm_hart_suspend(struct sbi_scratch *scratch, u32 suspend_type,
-			 ulong raddr, ulong rmode, ulong priv)
+			 ulong raddr, ulong rmode, ulong arg1)
 {
-	int oldstate, ret;
+	int ret;
 	const struct sbi_domain *dom = sbi_domain_thishart_ptr();
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
 
-	/* For now, we only allow suspend from S-mode or U-mode. */
-
 	/* Sanity check on domain assigned to current HART */
 	if (!dom)
-		return SBI_EINVAL;
+		return SBI_EFAIL;
 
 	/* Sanity check on suspend type */
 	if (SBI_HSM_SUSPEND_RET_DEFAULT < suspend_type &&
@@ -425,27 +481,26 @@ int sbi_hsm_hart_suspend(struct sbi_scratch *scratch, u32 suspend_type,
 
 	/* Additional sanity check for non-retentive suspend */
 	if (suspend_type & SBI_HSM_SUSP_NON_RET_BIT) {
+		/*
+		 * For now, we only allow non-retentive suspend from
+		 * S-mode or U-mode.
+		 */
 		if (rmode != PRV_S && rmode != PRV_U)
-			return SBI_EINVAL;
+			return SBI_EFAIL;
 		if (dom && !sbi_domain_check_addr(dom, raddr, rmode,
 						  SBI_DOMAIN_EXECUTE))
 			return SBI_EINVALID_ADDR;
 	}
 
 	/* Save the resume address and resume mode */
-	scratch->next_arg1 = priv;
+	scratch->next_arg1 = arg1;
 	scratch->next_addr = raddr;
 	scratch->next_mode = rmode;
 
 	/* Directly move from STARTED to SUSPENDED state */
-	oldstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_STARTED,
-				  SBI_HSM_STATE_SUSPENDED);
-	if (oldstate != SBI_HSM_STATE_STARTED) {
-		sbi_printf("%s: ERR: The hart is in invalid state [%u]\n",
-			   __func__, oldstate);
-		ret = SBI_EDENIED;
-		goto fail_restore_state;
-	}
+	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_STARTED,
+					 SBI_HSM_STATE_SUSPENDED))
+		return SBI_EFAIL;
 
 	/* Save the suspend type */
 	hdata->suspend_type = suspend_type;
@@ -480,18 +535,13 @@ int sbi_hsm_hart_suspend(struct sbi_scratch *scratch, u32 suspend_type,
 		jump_warmboot();
 	}
 
-fail_restore_state:
 	/*
 	 * We might have successfully resumed from retentive suspend
 	 * or suspend failed. In both cases, we restore state of hart.
 	 */
-	oldstate = atomic_cmpxchg(&hdata->state, SBI_HSM_STATE_SUSPENDED,
-				  SBI_HSM_STATE_STARTED);
-	if (oldstate != SBI_HSM_STATE_SUSPENDED) {
-		sbi_printf("%s: ERR: The hart is in invalid state [%u]\n",
-			   __func__, oldstate);
+	if (!__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_SUSPENDED,
+					 SBI_HSM_STATE_STARTED))
 		sbi_hart_hang();
-	}
 
 	return ret;
 }

lib/sbi/sbi_illegal_insn.c

@@ -90,7 +90,7 @@ static int system_opcode_insn(ulong insn, struct sbi_trap_regs *regs)
 		break;
 	default:
 		return truly_illegal_insn(insn, regs);
-	};
+	}
 
 	if (do_write && sbi_emulate_csr_write(csr_num, regs, new_csr_val))
 		return truly_illegal_insn(insn, regs);

lib/sbi/sbi_init.c

@@ -12,10 +12,12 @@
 #include <sbi/riscv_barrier.h>
 #include <sbi/riscv_locks.h>
 #include <sbi/sbi_console.h>
+#include <sbi/sbi_cppc.h>
 #include <sbi/sbi_domain.h>
 #include <sbi/sbi_ecall.h>
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_heap.h>
 #include <sbi/sbi_hsm.h>
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_irqchip.h>
@@ -69,6 +71,8 @@ static void sbi_boot_print_general(struct sbi_scratch *scratch)
 	const struct sbi_timer_device *tdev;
 	const struct sbi_console_device *cdev;
 	const struct sbi_system_reset_device *srdev;
+	const struct sbi_system_suspend_device *susp_dev;
+	const struct sbi_cppc_device *cppc_dev;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
 	if (scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS)
@@ -103,11 +107,32 @@ static void sbi_boot_print_general(struct sbi_scratch *scratch)
 	srdev = sbi_system_reset_get_device(SBI_SRST_RESET_TYPE_SHUTDOWN, 0);
 	sbi_printf("Platform Shutdown Device  : %s\n",
 		   (srdev) ? srdev->name : "---");
+	susp_dev = sbi_system_suspend_get_device();
+	sbi_printf("Platform Suspend Device   : %s\n",
+		   (susp_dev) ? susp_dev->name : "---");
+	cppc_dev = sbi_cppc_get_device();
+	sbi_printf("Platform CPPC Device      : %s\n",
+		   (cppc_dev) ? cppc_dev->name : "---");
 
 	/* Firmware details */
 	sbi_printf("Firmware Base             : 0x%lx\n", scratch->fw_start);
 	sbi_printf("Firmware Size             : %d KB\n",
 		   (u32)(scratch->fw_size / 1024));
+	sbi_printf("Firmware RW Offset        : 0x%lx\n", scratch->fw_rw_offset);
+	sbi_printf("Firmware RW Size          : %d KB\n",
+		   (u32)((scratch->fw_size - scratch->fw_rw_offset) / 1024));
+	sbi_printf("Firmware Heap Offset      : 0x%lx\n", scratch->fw_heap_offset);
+	sbi_printf("Firmware Heap Size        : "
+		   "%d KB (total), %d KB (reserved), %d KB (used), %d KB (free)\n",
+		   (u32)(scratch->fw_heap_size / 1024),
+		   (u32)(sbi_heap_reserved_space() / 1024),
+		   (u32)(sbi_heap_used_space() / 1024),
+		   (u32)(sbi_heap_free_space() / 1024));
+	sbi_printf("Firmware Scratch Size     : "
+		   "%d B (total), %d B (used), %d B (free)\n",
+		   SBI_SCRATCH_SIZE,
+		   (u32)sbi_scratch_used_space(),
+		   (u32)(SBI_SCRATCH_SIZE - sbi_scratch_used_space()));
 
 	/* SBI details */
 	sbi_printf("Runtime SBI Version       : %d.%d\n",
@@ -151,12 +176,13 @@ static void sbi_boot_print_hart(struct sbi_scratch *scratch, u32 hartid)
 	sbi_printf("Boot HART ISA Extensions  : %s\n", str);
 	sbi_printf("Boot HART PMP Count       : %d\n",
 		   sbi_hart_pmp_count(scratch));
-	sbi_printf("Boot HART PMP Granularity : %lu\n",
-		   sbi_hart_pmp_granularity(scratch));
+	sbi_printf("Boot HART PMP Granularity : %u bits\n",
+		   sbi_hart_pmp_log2gran(scratch));
 	sbi_printf("Boot HART PMP Address Bits: %d\n",
 		   sbi_hart_pmp_addrbits(scratch));
-	sbi_printf("Boot HART MHPM Count      : %d\n",
-		   sbi_hart_mhpm_count(scratch));
+	sbi_printf("Boot HART MHPM Info       : %lu (0x%08x)\n",
+		   sbi_popcount(sbi_hart_mhpm_mask(scratch)),
+		   sbi_hart_mhpm_mask(scratch));
 	sbi_hart_delegation_dump(scratch, "Boot HART ", "         ");
 }
 
@@ -169,6 +195,9 @@ static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
 {
 	unsigned long saved_mie, cmip;
 
+	if (__smp_load_acquire(&coldboot_done))
+		return;
+
 	/* Save MIE CSR */
 	saved_mie = csr_read(CSR_MIE);
 
@@ -179,7 +208,7 @@ static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
 	spin_lock(&coldboot_lock);
 
 	/* Mark current HART as waiting */
-	sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
+	sbi_hartmask_set_hartid(hartid, &coldboot_wait_hmask);
 
 	/* Release coldboot lock */
 	spin_unlock(&coldboot_lock);
@@ -190,13 +219,13 @@ static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
 			wfi();
 			cmip = csr_read(CSR_MIP);
 		 } while (!(cmip & (MIP_MSIP | MIP_MEIP)));
-	};
+	}
 
 	/* Acquire coldboot lock */
 	spin_lock(&coldboot_lock);
 
 	/* Unmark current HART as waiting */
-	sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
+	sbi_hartmask_clear_hartid(hartid, &coldboot_wait_hmask);
 
 	/* Release coldboot lock */
 	spin_unlock(&coldboot_lock);
@@ -216,6 +245,8 @@ static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
 
 static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
 {
+	u32 i, hartindex = sbi_hartid_to_hartindex(hartid);
+
 	/* Mark coldboot done */
 	__smp_store_release(&coldboot_done, 1);
 
@@ -223,22 +254,23 @@ static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
 	spin_lock(&coldboot_lock);
 
 	/* Send an IPI to all HARTs waiting for coldboot */
-	for (u32 i = 0; i <= sbi_scratch_last_hartid(); i++) {
-		if ((i != hartid) &&
-		    sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
-			sbi_ipi_raw_send(i);
+	sbi_hartmask_for_each_hartindex(i, &coldboot_wait_hmask) {
+		if (i == hartindex)
+			continue;
+		sbi_ipi_raw_send(i);
 	}
 
 	/* Release coldboot lock */
 	spin_unlock(&coldboot_lock);
 }
 
+static unsigned long entry_count_offset;
 static unsigned long init_count_offset;
 
 static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 {
 	int rc;
-	unsigned long *init_count;
+	unsigned long *count;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
 	/* Note: This has to be first thing in coldboot init sequence */
@@ -247,23 +279,35 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 		sbi_hart_hang();
 
 	/* Note: This has to be second thing in coldboot init sequence */
+	rc = sbi_heap_init(scratch);
+	if (rc)
+		sbi_hart_hang();
+
+	/* Note: This has to be the third thing in coldboot init sequence */
 	rc = sbi_domain_init(scratch, hartid);
 	if (rc)
 		sbi_hart_hang();
 
+	entry_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__);
+	if (!entry_count_offset)
+		sbi_hart_hang();
+
 	init_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__);
 	if (!init_count_offset)
 		sbi_hart_hang();
 
-	rc = sbi_hsm_init(scratch, hartid, TRUE);
+	count = sbi_scratch_offset_ptr(scratch, entry_count_offset);
+	(*count)++;
+
+	rc = sbi_hsm_init(scratch, hartid, true);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_platform_early_init(plat, TRUE);
+	rc = sbi_platform_early_init(plat, true);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_hart_init(scratch, TRUE);
+	rc = sbi_hart_init(scratch, true);
 	if (rc)
 		sbi_hart_hang();
 
@@ -271,43 +315,40 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_pmu_init(scratch, TRUE);
-	if (rc)
+	rc = sbi_pmu_init(scratch, true);
+	if (rc) {
+		sbi_printf("%s: pmu init failed (error %d)\n",
+			   __func__, rc);
 		sbi_hart_hang();
+	}
 
 	sbi_boot_print_banner(scratch);
 
-	rc = sbi_irqchip_init(scratch, TRUE);
+	rc = sbi_irqchip_init(scratch, true);
 	if (rc) {
 		sbi_printf("%s: irqchip init failed (error %d)\n",
 			   __func__, rc);
 		sbi_hart_hang();
 	}
 
-	rc = sbi_ipi_init(scratch, TRUE);
+	rc = sbi_ipi_init(scratch, true);
 	if (rc) {
 		sbi_printf("%s: ipi init failed (error %d)\n", __func__, rc);
 		sbi_hart_hang();
 	}
 
-	rc = sbi_tlb_init(scratch, TRUE);
+	rc = sbi_tlb_init(scratch, true);
 	if (rc) {
 		sbi_printf("%s: tlb init failed (error %d)\n", __func__, rc);
 		sbi_hart_hang();
 	}
 
-	rc = sbi_timer_init(scratch, TRUE);
+	rc = sbi_timer_init(scratch, true);
 	if (rc) {
 		sbi_printf("%s: timer init failed (error %d)\n", __func__, rc);
 		sbi_hart_hang();
 	}
 
-	rc = sbi_ecall_init();
-	if (rc) {
-		sbi_printf("%s: ecall init failed (error %d)\n", __func__, rc);
-		sbi_hart_hang();
-	}
-
 	/*
 	 * Note: Finalize domains after HSM initialization so that we
 	 * can startup non-root domains.
@@ -321,21 +362,26 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 		sbi_hart_hang();
 	}
 
-	rc = sbi_hart_pmp_configure(scratch);
+	/*
+	 * Note: Platform final initialization should be after finalizing
+	 * domains so that it sees correct domain assignment and PMP
+	 * configuration for FDT fixups.
+	 */
+	rc = sbi_platform_final_init(plat, true);
 	if (rc) {
-		sbi_printf("%s: PMP configure failed (error %d)\n",
+		sbi_printf("%s: platform final init failed (error %d)\n",
 			   __func__, rc);
 		sbi_hart_hang();
 	}
 
 	/*
-	 * Note: Platform final initialization should be last so that
-	 * it sees correct domain assignment and PMP configuration.
+	 * Note: Ecall initialization should be after platform final
+	 * initialization so that all available platform devices are
+	 * already registered.
 	 */
-	rc = sbi_platform_final_init(plat, TRUE);
+	rc = sbi_ecall_init();
 	if (rc) {
-		sbi_printf("%s: platform final init failed (error %d)\n",
-			   __func__, rc);
+		sbi_printf("%s: ecall init failed (error %d)\n", __func__, rc);
 		sbi_hart_hang();
 	}
 
@@ -345,72 +391,90 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 
 	sbi_boot_print_hart(scratch, hartid);
 
+	/*
+	 * Configure PMP at last because if SMEPMP is detected,
+	 * M-mode access to the S/U space will be rescinded.
+	 */
+	rc = sbi_hart_pmp_configure(scratch);
+	if (rc) {
+		sbi_printf("%s: PMP configure failed (error %d)\n",
+			   __func__, rc);
+		sbi_hart_hang();
+	}
+
 	wake_coldboot_harts(scratch, hartid);
 
-	init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
-	(*init_count)++;
+	count = sbi_scratch_offset_ptr(scratch, init_count_offset);
+	(*count)++;
 
-	sbi_hsm_prepare_next_jump(scratch, hartid);
-	sbi_hart_switch_mode(hartid, scratch->next_arg1, scratch->next_addr,
-			     scratch->next_mode, FALSE);
+	sbi_hsm_hart_start_finish(scratch, hartid);
 }
 
-static void init_warm_startup(struct sbi_scratch *scratch, u32 hartid)
+static void __noreturn init_warm_startup(struct sbi_scratch *scratch,
+					 u32 hartid)
 {
 	int rc;
-	unsigned long *init_count;
+	unsigned long *count;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	if (!init_count_offset)
+	if (!entry_count_offset || !init_count_offset)
 		sbi_hart_hang();
 
-	rc = sbi_hsm_init(scratch, hartid, FALSE);
+	count = sbi_scratch_offset_ptr(scratch, entry_count_offset);
+	(*count)++;
+
+	rc = sbi_hsm_init(scratch, hartid, false);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_platform_early_init(plat, FALSE);
+	rc = sbi_platform_early_init(plat, false);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_hart_init(scratch, FALSE);
+	rc = sbi_hart_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_pmu_init(scratch, FALSE);
+	rc = sbi_pmu_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_irqchip_init(scratch, FALSE);
+	rc = sbi_irqchip_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_ipi_init(scratch, FALSE);
+	rc = sbi_ipi_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_tlb_init(scratch, FALSE);
+	rc = sbi_tlb_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_timer_init(scratch, FALSE);
+	rc = sbi_timer_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
+	rc = sbi_platform_final_init(plat, false);
+	if (rc)
+		sbi_hart_hang();
+
+	/*
+	 * Configure PMP at last because if SMEPMP is detected,
+	 * M-mode access to the S/U space will be rescinded.
+	 */
 	rc = sbi_hart_pmp_configure(scratch);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_platform_final_init(plat, FALSE);
-	if (rc)
-		sbi_hart_hang();
+	count = sbi_scratch_offset_ptr(scratch, init_count_offset);
+	(*count)++;
 
-	init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
-	(*init_count)++;
-
-	sbi_hsm_prepare_next_jump(scratch, hartid);
+	sbi_hsm_hart_start_finish(scratch, hartid);
 }
 
-static void init_warm_resume(struct sbi_scratch *scratch)
+static void __noreturn init_warm_resume(struct sbi_scratch *scratch,
+					u32 hartid)
 {
 	int rc;
 
@@ -424,7 +488,7 @@ static void init_warm_resume(struct sbi_scratch *scratch)
 	if (rc)
 		sbi_hart_hang();
 
-	sbi_hsm_hart_resume_finish(scratch);
+	sbi_hsm_hart_resume_finish(scratch, hartid);
 }
 
 static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
@@ -437,14 +501,12 @@ static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
 	if (hstate < 0)
 		sbi_hart_hang();
 
-	if (hstate == SBI_HSM_STATE_SUSPENDED)
-		init_warm_resume(scratch);
-	else
+	if (hstate == SBI_HSM_STATE_SUSPENDED) {
+		init_warm_resume(scratch, hartid);
+	} else {
+		sbi_ipi_raw_clear(sbi_hartid_to_hartindex(hartid));
 		init_warm_startup(scratch, hartid);
-
-	sbi_hart_switch_mode(hartid, scratch->next_arg1,
-			     scratch->next_addr,
-			     scratch->next_mode, FALSE);
+	}
 }
 
 static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
@@ -463,26 +525,32 @@ static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
  */
 void __noreturn sbi_init(struct sbi_scratch *scratch)
 {
-	bool next_mode_supported	= FALSE;
-	bool coldboot			= FALSE;
+	u32 i, h;
+	bool hartid_valid		= false;
+	bool next_mode_supported	= false;
+	bool coldboot			= false;
 	u32 hartid			= current_hartid();
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
-	    sbi_platform_hart_invalid(plat, hartid))
+	for (i = 0; i < plat->hart_count; i++) {
+		h = (plat->hart_index2id) ? plat->hart_index2id[i] : i;
+		if (h == hartid)
+			hartid_valid = true;
+	}
+	if (!hartid_valid)
 		sbi_hart_hang();
 
 	switch (scratch->next_mode) {
 	case PRV_M:
-		next_mode_supported = TRUE;
+		next_mode_supported = true;
 		break;
 	case PRV_S:
 		if (misa_extension('S'))
-			next_mode_supported = TRUE;
+			next_mode_supported = true;
 		break;
 	case PRV_U:
 		if (misa_extension('U'))
-			next_mode_supported = TRUE;
+			next_mode_supported = true;
 		break;
 	default:
 		sbi_hart_hang();
@@ -498,8 +566,11 @@ void __noreturn sbi_init(struct sbi_scratch *scratch)
 	 * HARTs which satisfy above condition.
 	 */
 
-	if (next_mode_supported && atomic_xchg(&coldboot_lottery, 1) == 0)
-		coldboot = TRUE;
+	if (sbi_platform_cold_boot_allowed(plat, hartid)) {
+		if (next_mode_supported &&
+		    atomic_xchg(&coldboot_lottery, 1) == 0)
+			coldboot = true;
+	}
 
 	/*
 	 * Do platform specific nascent (very early) initialization so
@@ -515,6 +586,23 @@ void __noreturn sbi_init(struct sbi_scratch *scratch)
 		init_warmboot(scratch, hartid);
 }
 
+unsigned long sbi_entry_count(u32 hartid)
+{
+	struct sbi_scratch *scratch;
+	unsigned long *entry_count;
+
+	if (!entry_count_offset)
+		return 0;
+
+	scratch = sbi_hartid_to_scratch(hartid);
+	if (!scratch)
+		return 0;
+
+	entry_count = sbi_scratch_offset_ptr(scratch, entry_count_offset);
+
+	return *entry_count;
+}
+
 unsigned long sbi_init_count(u32 hartid)
 {
 	struct sbi_scratch *scratch;
@@ -546,7 +634,7 @@ void __noreturn sbi_exit(struct sbi_scratch *scratch)
 	u32 hartid			= current_hartid();
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	if (sbi_platform_hart_invalid(plat, hartid))
+	if (!sbi_hartid_valid(hartid))
 		sbi_hart_hang();
 
 	sbi_platform_early_exit(plat);

lib/sbi/sbi_ipi.c

@@ -27,14 +27,19 @@ struct sbi_ipi_data {
 	unsigned long ipi_type;
 };
 
+_Static_assert(
+	8 * sizeof(((struct sbi_ipi_data*)0)->ipi_type) == SBI_IPI_EVENT_MAX,
+	"type of sbi_ipi_data.ipi_type has changed, please redefine SBI_IPI_EVENT_MAX"
+	);
+
 static unsigned long ipi_data_off;
 static const struct sbi_ipi_device *ipi_dev = NULL;
 static const struct sbi_ipi_event_ops *ipi_ops_array[SBI_IPI_EVENT_MAX];
 
-static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
+static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartindex,
 			u32 event, void *data)
 {
-	int ret;
+	int ret = 0;
 	struct sbi_scratch *remote_scratch = NULL;
 	struct sbi_ipi_data *ipi_data;
 	const struct sbi_ipi_event_ops *ipi_ops;
@@ -44,7 +49,7 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
 		return SBI_EINVAL;
 	ipi_ops = ipi_ops_array[event];
 
-	remote_scratch = sbi_hartid_to_scratch(remote_hartid);
+	remote_scratch = sbi_hartindex_to_scratch(remote_hartindex);
 	if (!remote_scratch)
 		return SBI_EINVAL;
 
@@ -52,23 +57,45 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
 
 	if (ipi_ops->update) {
 		ret = ipi_ops->update(scratch, remote_scratch,
-				      remote_hartid, data);
-		if (ret < 0)
+				      remote_hartindex, data);
+		if (ret != SBI_IPI_UPDATE_SUCCESS)
 			return ret;
+	} else if (scratch == remote_scratch) {
+		/*
+		 * IPI events with an update() callback are expected to return
+		 * SBI_IPI_UPDATE_BREAK for self-IPIs. For other events, check
+		 * for self-IPI and execute the callback directly here.
+		 */
+		ipi_ops->process(scratch);
+		return 0;
 	}
 
 	/*
 	 * Set IPI type on remote hart's scratch area and
-	 * trigger the interrupt
+	 * trigger the interrupt.
+	 *
+	 * Multiple harts may be trying to send IPI to the
+	 * remote hart so call sbi_ipi_raw_send() only when
+	 * the ipi_type was previously zero.
 	 */
-	atomic_raw_set_bit(event, &ipi_data->ipi_type);
-	smp_wmb();
-
-	if (ipi_dev && ipi_dev->ipi_send)
-		ipi_dev->ipi_send(remote_hartid);
+	if (!__atomic_fetch_or(&ipi_data->ipi_type,
+				BIT(event), __ATOMIC_RELAXED))
+		ret = sbi_ipi_raw_send(remote_hartindex);
 
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_SENT);
 
+	return ret;
+}
+
+static int sbi_ipi_sync(struct sbi_scratch *scratch, u32 event)
+{
+	const struct sbi_ipi_event_ops *ipi_ops;
+
+	if ((SBI_IPI_EVENT_MAX <= event) ||
+	    !ipi_ops_array[event])
+		return SBI_EINVAL;
+	ipi_ops = ipi_ops_array[event];
+
 	if (ipi_ops->sync)
 		ipi_ops->sync(scratch);
 
@@ -82,35 +109,55 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
  */
 int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
 {
-	int rc;
+	int rc = 0;
+	bool retry_needed;
 	ulong i, m;
+	struct sbi_hartmask target_mask = {0};
 	struct sbi_domain *dom = sbi_domain_thishart_ptr();
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 
+	/* Find the target harts */
 	if (hbase != -1UL) {
 		rc = sbi_hsm_hart_interruptible_mask(dom, hbase, &m);
 		if (rc)
 			return rc;
 		m &= hmask;
 
-		/* Send IPIs */
 		for (i = hbase; m; i++, m >>= 1) {
 			if (m & 1UL)
-				sbi_ipi_send(scratch, i, event, data);
+				sbi_hartmask_set_hartid(i, &target_mask);
 		}
 	} else {
 		hbase = 0;
 		while (!sbi_hsm_hart_interruptible_mask(dom, hbase, &m)) {
-			/* Send IPIs */
 			for (i = hbase; m; i++, m >>= 1) {
 				if (m & 1UL)
-					sbi_ipi_send(scratch, i, event, data);
+					sbi_hartmask_set_hartid(i, &target_mask);
 			}
 			hbase += BITS_PER_LONG;
 		}
 	}
 
-	return 0;
+	/* Send IPIs */
+	do {
+		retry_needed = false;
+		sbi_hartmask_for_each_hartindex(i, &target_mask) {
+			rc = sbi_ipi_send(scratch, i, event, data);
+			if (rc < 0)
+				goto done;
+			if (rc == SBI_IPI_UPDATE_RETRY)
+				retry_needed = true;
+			else
+				sbi_hartmask_clear_hartindex(i, &target_mask);
+			rc = 0;
+		}
+	} while (retry_needed);
+
+done:
+	/* Sync IPIs */
+	sbi_ipi_sync(scratch, event);
+
+	return rc;
 }
 
 int sbi_ipi_event_create(const struct sbi_ipi_event_ops *ops)
@@ -163,7 +210,7 @@ void sbi_ipi_clear_smode(void)
 
 static void sbi_ipi_process_halt(struct sbi_scratch *scratch)
 {
-	sbi_hsm_hart_stop(scratch, TRUE);
+	sbi_hsm_hart_stop(scratch, true);
 }
 
 static struct sbi_ipi_event_ops ipi_halt_ops = {
@@ -186,37 +233,61 @@ void sbi_ipi_process(void)
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 	struct sbi_ipi_data *ipi_data =
 			sbi_scratch_offset_ptr(scratch, ipi_data_off);
-	u32 hartid = current_hartid();
+	u32 hartindex = sbi_hartid_to_hartindex(current_hartid());
 
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_RECVD);
-	if (ipi_dev && ipi_dev->ipi_clear)
-		ipi_dev->ipi_clear(hartid);
+	sbi_ipi_raw_clear(hartindex);
 
 	ipi_type = atomic_raw_xchg_ulong(&ipi_data->ipi_type, 0);
 	ipi_event = 0;
 	while (ipi_type) {
-		if (!(ipi_type & 1UL))
-			goto skip;
-
-		ipi_ops = ipi_ops_array[ipi_event];
-		if (ipi_ops && ipi_ops->process)
-			ipi_ops->process(scratch);
-
-skip:
+		if (ipi_type & 1UL) {
+			ipi_ops = ipi_ops_array[ipi_event];
+			if (ipi_ops)
+				ipi_ops->process(scratch);
+		}
 		ipi_type = ipi_type >> 1;
 		ipi_event++;
-	};
+	}
 }
 
-int sbi_ipi_raw_send(u32 target_hart)
+int sbi_ipi_raw_send(u32 hartindex)
 {
 	if (!ipi_dev || !ipi_dev->ipi_send)
 		return SBI_EINVAL;
 
-	ipi_dev->ipi_send(target_hart);
+	/*
+	 * Ensure that memory or MMIO writes done before
+	 * this function are not observed after the memory
+	 * or MMIO writes done by the ipi_send() device
+	 * callback. This also allows the ipi_send() device
+	 * callback to use relaxed MMIO writes.
+	 *
+	 * This pairs with the wmb() in sbi_ipi_raw_clear().
+	 */
+	wmb();
+
+	ipi_dev->ipi_send(hartindex);
 	return 0;
 }
 
+void sbi_ipi_raw_clear(u32 hartindex)
+{
+	if (ipi_dev && ipi_dev->ipi_clear)
+		ipi_dev->ipi_clear(hartindex);
+
+	/*
+	 * Ensure that memory or MMIO writes after this
+	 * function returns are not observed before the
+	 * memory or MMIO writes done by the ipi_clear()
+	 * device callback. This also allows ipi_clear()
+	 * device callback to use relaxed MMIO writes.
+	 *
+	 * This pairs with the wmb() in sbi_ipi_raw_send().
+	 */
+	wmb();
+}
+
 const struct sbi_ipi_device *sbi_ipi_get_device(void)
 {
 	return ipi_dev;

lib/sbi/sbi_misaligned_ldst.c

@@ -211,16 +211,14 @@ int sbi_misaligned_store_handler(ulong addr, ulong tval2, ulong tinst,
 	} else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
 		len	       = 8;
 		val.data_ulong = GET_RS2S(insn, regs);
-	} else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
-		   ((insn >> SH_RD) & 0x1f)) {
+	} else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
 		len	       = 8;
 		val.data_ulong = GET_RS2C(insn, regs);
 #endif
 	} else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
 		len	       = 4;
 		val.data_ulong = GET_RS2S(insn, regs);
-	} else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
-		   ((insn >> SH_RD) & 0x1f)) {
+	} else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
 		len	       = 4;
 		val.data_ulong = GET_RS2C(insn, regs);
 #ifdef __riscv_flen

lib/sbi/sbi_platform.c

@@ -71,20 +71,3 @@ done:
 	else
 		sbi_strncpy(features_str, "none", nfstr);
 }
-
-u32 sbi_platform_hart_index(const struct sbi_platform *plat, u32 hartid)
-{
-	u32 i;
-
-	if (!plat)
-		return -1U;
-	if (plat->hart_index2id) {
-		for (i = 0; i < plat->hart_count; i++) {
-			if (plat->hart_index2id[i] == hartid)
-				return i;
-		}
-		return -1U;
-	}
-
-	return hartid;
-}

lib/sbi/sbi_pmu.c

@@ -12,7 +12,7 @@
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_ecall_interface.h>
 #include <sbi/sbi_hart.h>
-#include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_heap.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_pmu.h>
 #include <sbi/sbi_scratch.h>
@@ -50,23 +50,43 @@ union sbi_pmu_ctr_info {
 	};
 };
 
+#if SBI_PMU_FW_CTR_MAX >= BITS_PER_LONG
+#error "Can't handle firmware counters beyond BITS_PER_LONG"
+#endif
+
+/** Per-HART state of the PMU counters */
+struct sbi_pmu_hart_state {
+	/* HART to which this state belongs */
+	uint32_t hartid;
+	/* Counter to enabled event mapping */
+	uint32_t active_events[SBI_PMU_HW_CTR_MAX + SBI_PMU_FW_CTR_MAX];
+	/* Bitmap of firmware counters started */
+	unsigned long fw_counters_started;
+	/*
+	 * Counter values for SBI firmware events and event codes
+	 * for platform firmware events. Both are mutually exclusive
+	 * and hence can optimally share the same memory.
+	 */
+	uint64_t fw_counters_data[SBI_PMU_FW_CTR_MAX];
+};
+
+/** Offset of pointer to PMU HART state in scratch space */
+static unsigned long phs_ptr_offset;
+
+#define pmu_get_hart_state_ptr(__scratch)				\
+	sbi_scratch_read_type((__scratch), void *, phs_ptr_offset)
+
+#define pmu_thishart_state_ptr()					\
+	pmu_get_hart_state_ptr(sbi_scratch_thishart_ptr())
+
+#define pmu_set_hart_state_ptr(__scratch, __phs)			\
+	sbi_scratch_write_type((__scratch), void *, phs_ptr_offset, (__phs))
+
 /* Platform specific PMU device */
 static const struct sbi_pmu_device *pmu_dev = NULL;
 
 /* Mapping between event range and possible counters  */
-static struct sbi_pmu_hw_event hw_event_map[SBI_PMU_HW_EVENT_MAX] = {0};
-
-/* counter to enabled event mapping */
-static uint32_t active_events[SBI_HARTMASK_MAX_BITS][SBI_PMU_HW_CTR_MAX + SBI_PMU_FW_CTR_MAX];
-
-/* Bitmap of firmware counters started on each HART */
-#if SBI_PMU_FW_CTR_MAX >= BITS_PER_LONG
-#error "Can't handle firmware counters beyond BITS_PER_LONG"
-#endif
-static unsigned long fw_counters_started[SBI_HARTMASK_MAX_BITS];
-
-/* Values of firmwares counters on each HART */
-static uint64_t fw_counters_value[SBI_HARTMASK_MAX_BITS][SBI_PMU_FW_CTR_MAX] = {0};
+static struct sbi_pmu_hw_event *hw_event_map;
 
 /* Maximum number of hardware events available */
 static uint32_t num_hw_events;
@@ -77,7 +97,8 @@ static uint32_t num_hw_ctrs;
 static uint32_t total_ctrs;
 
 /* Helper macros to retrieve event idx and code type */
-#define get_cidx_type(x) ((x & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16)
+#define get_cidx_type(x) \
+  (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> SBI_PMU_EVENT_IDX_TYPE_OFFSET)
 #define get_cidx_code(x) (x & SBI_PMU_EVENT_IDX_CODE_MASK)
 
 /**
@@ -85,7 +106,7 @@ static uint32_t total_ctrs;
  * @param evtA Pointer to the existing hw event structure
  * @param evtB Pointer to the new hw event structure
  *
- * Return FALSE if the range doesn't overlap, TRUE otherwise
+ * Return false if the range doesn't overlap, true otherwise
  */
 static bool pmu_event_range_overlap(struct sbi_pmu_hw_event *evtA,
 				    struct sbi_pmu_hw_event *evtB)
@@ -93,20 +114,21 @@ static bool pmu_event_range_overlap(struct sbi_pmu_hw_event *evtA,
 	/* check if the range of events overlap with a previous entry */
 	if (((evtA->end_idx < evtB->start_idx) && (evtA->end_idx < evtB->end_idx)) ||
 	   ((evtA->start_idx > evtB->start_idx) && (evtA->start_idx > evtB->end_idx)))
-		return FALSE;
-	return TRUE;
+		return false;
+	return true;
 }
 
 static bool pmu_event_select_overlap(struct sbi_pmu_hw_event *evt,
 				     uint64_t select_val, uint64_t select_mask)
 {
 	if ((evt->select == select_val) && (evt->select_mask == select_mask))
-		return TRUE;
+		return true;
 
-	return FALSE;
+	return false;
 }
 
-static int pmu_event_validate(unsigned long event_idx)
+static int pmu_event_validate(struct sbi_pmu_hart_state *phs,
+			      unsigned long event_idx, uint64_t edata)
 {
 	uint32_t event_idx_type = get_cidx_type(event_idx);
 	uint32_t event_idx_code = get_cidx_code(event_idx);
@@ -118,9 +140,15 @@ static int pmu_event_validate(unsigned long event_idx)
 		event_idx_code_max = SBI_PMU_HW_GENERAL_MAX;
 		break;
 	case SBI_PMU_EVENT_TYPE_FW:
-		if (SBI_PMU_FW_MAX <= event_idx_code &&
-		    pmu_dev && pmu_dev->fw_event_validate_code)
-			return pmu_dev->fw_event_validate_code(event_idx_code);
+		if ((event_idx_code >= SBI_PMU_FW_MAX &&
+		    event_idx_code <= SBI_PMU_FW_RESERVED_MAX) ||
+		    event_idx_code > SBI_PMU_FW_PLATFORM)
+			return SBI_EINVAL;
+
+		if (SBI_PMU_FW_PLATFORM == event_idx_code &&
+		    pmu_dev && pmu_dev->fw_event_validate_encoding)
+			return pmu_dev->fw_event_validate_encoding(phs->hartid,
+							           edata);
 		else
 			event_idx_code_max = SBI_PMU_FW_MAX;
 		break;
@@ -153,16 +181,16 @@ static int pmu_event_validate(unsigned long event_idx)
 	return SBI_EINVAL;
 }
 
-static int pmu_ctr_validate(uint32_t cidx, uint32_t *event_idx_code)
+static int pmu_ctr_validate(struct sbi_pmu_hart_state *phs,
+			    uint32_t cidx, uint32_t *event_idx_code)
 {
 	uint32_t event_idx_val;
 	uint32_t event_idx_type;
-	u32 hartid = current_hartid();
 
 	if (cidx >= total_ctrs)
 		return SBI_EINVAL;
 
-	event_idx_val = active_events[hartid][cidx];
+	event_idx_val = phs->active_events[cidx];
 	event_idx_type = get_cidx_type(event_idx_val);
 	if (event_idx_val == SBI_PMU_EVENT_IDX_INVALID ||
 	    event_idx_type >= SBI_PMU_EVENT_TYPE_MAX)
@@ -177,18 +205,26 @@ int sbi_pmu_ctr_fw_read(uint32_t cidx, uint64_t *cval)
 {
 	int event_idx_type;
 	uint32_t event_code;
-	u32 hartid = current_hartid();
+	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
 
-	event_idx_type = pmu_ctr_validate(cidx, &event_code);
+	event_idx_type = pmu_ctr_validate(phs, cidx, &event_code);
 	if (event_idx_type != SBI_PMU_EVENT_TYPE_FW)
 		return SBI_EINVAL;
 
-	if (SBI_PMU_FW_MAX <= event_code &&
-	    pmu_dev && pmu_dev->fw_counter_read_value)
-		fw_counters_value[hartid][cidx - num_hw_ctrs] =
-			pmu_dev->fw_counter_read_value(cidx - num_hw_ctrs);
+	if ((event_code >= SBI_PMU_FW_MAX &&
+	    event_code <= SBI_PMU_FW_RESERVED_MAX) ||
+	    event_code > SBI_PMU_FW_PLATFORM)
+		return SBI_EINVAL;
 
-	*cval = fw_counters_value[hartid][cidx - num_hw_ctrs];
+	if (SBI_PMU_FW_PLATFORM == event_code) {
+		if (pmu_dev && pmu_dev->fw_counter_read_value)
+			*cval = pmu_dev->fw_counter_read_value(phs->hartid,
+							       cidx -
+							       num_hw_ctrs);
+		else
+			*cval = 0;
+	} else
+		*cval = phs->fw_counters_data[cidx - num_hw_ctrs];
 
 	return 0;
 }
@@ -200,8 +236,7 @@ static int pmu_add_hw_event_map(u32 eidx_start, u32 eidx_end, u32 cmap,
 	bool is_overlap;
 	struct sbi_pmu_hw_event *event = &hw_event_map[num_hw_events];
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
-	int hw_ctr_avail = sbi_hart_mhpm_count(scratch);
-	uint32_t ctr_avail_mask = ((uint32_t)(~0) >> (32 - (hw_ctr_avail + 3)));
+	uint32_t ctr_avail_mask = sbi_hart_mhpm_mask(scratch) | 0x7;
 
 	/* The first two counters are reserved by priv spec */
 	if (eidx_start > SBI_PMU_HW_INSTRUCTIONS && (cmap & SBI_PMU_FIXED_CTR_MASK))
@@ -318,8 +353,11 @@ static int pmu_ctr_start_hw(uint32_t cidx, uint64_t ival, bool ival_update)
 	if (cidx >= num_hw_ctrs || cidx == 1)
 		return SBI_EINVAL;
 
-	if (sbi_hart_priv_version(scratch) < SBI_HART_PRIV_VER_1_11)
-		goto skip_inhibit_update;
+	if (sbi_hart_priv_version(scratch) < SBI_HART_PRIV_VER_1_11) {
+		if (ival_update)
+			pmu_ctr_write_hw(cidx, ival);
+		return 0;
+	}
 
 	/*
 	 * Some of the hardware may not support mcountinhibit but perf stat
@@ -333,13 +371,12 @@ static int pmu_ctr_start_hw(uint32_t cidx, uint64_t ival, bool ival_update)
 
 	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
 		pmu_ctr_enable_irq_hw(cidx);
-	if (pmu_dev && pmu_dev->hw_counter_enable_irq)
-		pmu_dev->hw_counter_enable_irq(cidx);
-	csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
-
-skip_inhibit_update:
 	if (ival_update)
 		pmu_ctr_write_hw(cidx, ival);
+	if (pmu_dev && pmu_dev->hw_counter_enable_irq)
+		pmu_dev->hw_counter_enable_irq(cidx);
+
+	csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
 
 	return 0;
 }
@@ -356,24 +393,37 @@ int sbi_pmu_irq_bit(void)
 	return 0;
 }
 
-static int pmu_ctr_start_fw(uint32_t cidx, uint32_t event_code,
-			    uint64_t ival, bool ival_update)
+static int pmu_ctr_start_fw(struct sbi_pmu_hart_state *phs,
+			    uint32_t cidx, uint32_t event_code,
+			    uint64_t event_data, uint64_t ival,
+			    bool ival_update)
 {
-	int ret;
-	u32 hartid = current_hartid();
+	if ((event_code >= SBI_PMU_FW_MAX &&
+	    event_code <= SBI_PMU_FW_RESERVED_MAX) ||
+	    event_code > SBI_PMU_FW_PLATFORM)
+		return SBI_EINVAL;
 
-	if (SBI_PMU_FW_MAX <= event_code &&
-	    pmu_dev && pmu_dev->fw_counter_start) {
-		ret = pmu_dev->fw_counter_start(cidx - num_hw_ctrs,
-						event_code,
-						ival, ival_update);
-		if (ret)
-			return ret;
+	if (SBI_PMU_FW_PLATFORM == event_code) {
+		if (!pmu_dev ||
+		    !pmu_dev->fw_counter_write_value ||
+		    !pmu_dev->fw_counter_start) {
+			return SBI_EINVAL;
+		    }
+
+		if (ival_update)
+			pmu_dev->fw_counter_write_value(phs->hartid,
+							cidx - num_hw_ctrs,
+							ival);
+
+		return pmu_dev->fw_counter_start(phs->hartid,
+						 cidx - num_hw_ctrs,
+						 event_data);
+	} else {
+		if (ival_update)
+			phs->fw_counters_data[cidx - num_hw_ctrs] = ival;
 	}
 
-	if (ival_update)
-		fw_counters_value[hartid][cidx - num_hw_ctrs] = ival;
-	fw_counters_started[hartid] |= BIT(cidx - num_hw_ctrs);
+	phs->fw_counters_started |= BIT(cidx - num_hw_ctrs);
 
 	return 0;
 }
@@ -381,26 +431,36 @@ static int pmu_ctr_start_fw(uint32_t cidx, uint32_t event_code,
 int sbi_pmu_ctr_start(unsigned long cbase, unsigned long cmask,
 		      unsigned long flags, uint64_t ival)
 {
+	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
 	int event_idx_type;
 	uint32_t event_code;
 	int ret = SBI_EINVAL;
-	bool bUpdate = FALSE;
+	bool bUpdate = false;
 	int i, cidx;
+	uint64_t edata;
 
 	if ((cbase + sbi_fls(cmask)) >= total_ctrs)
 		return ret;
 
-	if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE)
-		bUpdate = TRUE;
+	if (flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT)
+		return SBI_ENO_SHMEM;
 
-	for_each_set_bit(i, &cmask, total_ctrs) {
+	if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE)
+		bUpdate = true;
+
+	for_each_set_bit(i, &cmask, BITS_PER_LONG) {
 		cidx = i + cbase;
-		event_idx_type = pmu_ctr_validate(cidx, &event_code);
+		event_idx_type = pmu_ctr_validate(phs, cidx, &event_code);
 		if (event_idx_type < 0)
 			/* Continue the start operation for other counters */
 			continue;
-		else if (event_idx_type == SBI_PMU_EVENT_TYPE_FW)
-			ret = pmu_ctr_start_fw(cidx, event_code, ival, bUpdate);
+		else if (event_idx_type == SBI_PMU_EVENT_TYPE_FW) {
+			edata = (event_code == SBI_PMU_FW_PLATFORM) ?
+				 phs->fw_counters_data[cidx - num_hw_ctrs]
+				 : 0x0;
+			ret = pmu_ctr_start_fw(phs, cidx, event_code, edata,
+					       ival, bUpdate);
+		}
 		else
 			ret = pmu_ctr_start_hw(cidx, ival, bUpdate);
 	}
@@ -425,23 +485,32 @@ static int pmu_ctr_stop_hw(uint32_t cidx)
 	if (!__test_bit(cidx, &mctr_inhbt)) {
 		__set_bit(cidx, &mctr_inhbt);
 		csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
+		if (pmu_dev && pmu_dev->hw_counter_disable_irq) {
+			pmu_dev->hw_counter_disable_irq(cidx);
+		}
 		return 0;
 	} else
 		return SBI_EALREADY_STOPPED;
 }
 
-static int pmu_ctr_stop_fw(uint32_t cidx, uint32_t event_code)
+static int pmu_ctr_stop_fw(struct sbi_pmu_hart_state *phs,
+			   uint32_t cidx, uint32_t event_code)
 {
 	int ret;
 
-	if (SBI_PMU_FW_MAX <= event_code &&
+	if ((event_code >= SBI_PMU_FW_MAX &&
+	    event_code <= SBI_PMU_FW_RESERVED_MAX) ||
+	    event_code > SBI_PMU_FW_PLATFORM)
+		return SBI_EINVAL;
+
+	if (SBI_PMU_FW_PLATFORM == event_code &&
 	    pmu_dev && pmu_dev->fw_counter_stop) {
-		ret = pmu_dev->fw_counter_stop(cidx - num_hw_ctrs);
+		ret = pmu_dev->fw_counter_stop(phs->hartid, cidx - num_hw_ctrs);
 		if (ret)
 			return ret;
 	}
 
-	fw_counters_started[current_hartid()] &= ~BIT(cidx - num_hw_ctrs);
+	phs->fw_counters_started &= ~BIT(cidx - num_hw_ctrs);
 
 	return 0;
 }
@@ -465,7 +534,7 @@ static int pmu_reset_hw_mhpmevent(int ctr_idx)
 int sbi_pmu_ctr_stop(unsigned long cbase, unsigned long cmask,
 		     unsigned long flag)
 {
-	u32 hartid = current_hartid();
+	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
 	int ret = SBI_EINVAL;
 	int event_idx_type;
 	uint32_t event_code;
@@ -474,20 +543,23 @@ int sbi_pmu_ctr_stop(unsigned long cbase, unsigned long cmask,
 	if ((cbase + sbi_fls(cmask)) >= total_ctrs)
 		return SBI_EINVAL;
 
-	for_each_set_bit(i, &cmask, total_ctrs) {
+	if (flag & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT)
+		return SBI_ENO_SHMEM;
+
+	for_each_set_bit(i, &cmask, BITS_PER_LONG) {
 		cidx = i + cbase;
-		event_idx_type = pmu_ctr_validate(cidx, &event_code);
+		event_idx_type = pmu_ctr_validate(phs, cidx, &event_code);
 		if (event_idx_type < 0)
 			/* Continue the stop operation for other counters */
 			continue;
 
 		else if (event_idx_type == SBI_PMU_EVENT_TYPE_FW)
-			ret = pmu_ctr_stop_fw(cidx, event_code);
+			ret = pmu_ctr_stop_fw(phs, cidx, event_code);
 		else
 			ret = pmu_ctr_stop_hw(cidx);
 
-		if (flag & SBI_PMU_STOP_FLAG_RESET) {
-			active_events[hartid][cidx] = SBI_PMU_EVENT_IDX_INVALID;
+		if (cidx > (CSR_INSTRET - CSR_CYCLE) && flag & SBI_PMU_STOP_FLAG_RESET) {
+			phs->active_events[cidx] = SBI_PMU_EVENT_IDX_INVALID;
 			pmu_reset_hw_mhpmevent(cidx);
 		}
 	}
@@ -533,7 +605,10 @@ static int pmu_update_hw_mhpmevent(struct sbi_pmu_hw_event *hw_evt, int ctr_idx,
 		pmu_dev->hw_counter_disable_irq(ctr_idx);
 
 	/* Update the inhibit flags based on inhibit flags received from supervisor */
-	pmu_update_inhibit_flags(flags, &mhpmevent_val);
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
+		pmu_update_inhibit_flags(flags, &mhpmevent_val);
+	if (pmu_dev && pmu_dev->hw_counter_filter_mode)
+		pmu_dev->hw_counter_filter_mode(flags, ctr_idx);
 
 #if __riscv_xlen == 32
 	csr_write_num(CSR_MHPMEVENT3 + ctr_idx - 3, mhpmevent_val & 0xFFFFFFFF);
@@ -547,7 +622,50 @@ static int pmu_update_hw_mhpmevent(struct sbi_pmu_hw_event *hw_evt, int ctr_idx,
 	return 0;
 }
 
-static int pmu_ctr_find_fixed_fw(unsigned long evt_idx_code)
+static int pmu_fixed_ctr_update_inhibit_bits(int fixed_ctr, unsigned long flags)
+{
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	uint64_t cfg_val = 0, cfg_csr_no;
+#if __riscv_xlen == 32
+	uint64_t cfgh_csr_no;
+#endif
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMCNTRPMF) &&
+		!(pmu_dev && pmu_dev->hw_counter_filter_mode))
+		return fixed_ctr;
+
+	switch (fixed_ctr) {
+	case 0:
+		cfg_csr_no = CSR_MCYCLECFG;
+#if __riscv_xlen == 32
+		cfgh_csr_no = CSR_MCYCLECFGH;
+#endif
+		break;
+	case 2:
+		cfg_csr_no = CSR_MINSTRETCFG;
+#if __riscv_xlen == 32
+		cfgh_csr_no = CSR_MINSTRETCFGH;
+#endif
+		break;
+	default:
+		return SBI_EFAIL;
+	}
+
+	cfg_val |= MHPMEVENT_MINH;
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMCNTRPMF)) {
+		pmu_update_inhibit_flags(flags, &cfg_val);
+#if __riscv_xlen == 32
+		csr_write_num(cfg_csr_no, cfg_val & 0xFFFFFFFF);
+		csr_write_num(cfgh_csr_no, cfg_val >> BITS_PER_LONG);
+#else
+		csr_write_num(cfg_csr_no, cfg_val);
+#endif
+	}
+	if (pmu_dev && pmu_dev->hw_counter_filter_mode)
+		pmu_dev->hw_counter_filter_mode(flags, fixed_ctr);
+	return fixed_ctr;
+}
+
+static int pmu_ctr_find_fixed_hw(unsigned long evt_idx_code)
 {
 	/* Non-programmables counters are enabled always. No need to do lookup */
 	if (evt_idx_code == SBI_PMU_HW_CPU_CYCLES)
@@ -558,14 +676,15 @@ static int pmu_ctr_find_fixed_fw(unsigned long evt_idx_code)
 		return SBI_EINVAL;
 }
 
-static int pmu_ctr_find_hw(unsigned long cbase, unsigned long cmask, unsigned long flags,
+static int pmu_ctr_find_hw(struct sbi_pmu_hart_state *phs,
+			   unsigned long cbase, unsigned long cmask,
+			   unsigned long flags,
 			   unsigned long event_idx, uint64_t data)
 {
 	unsigned long ctr_mask;
 	int i, ret = 0, fixed_ctr, ctr_idx = SBI_ENOTSUPP;
 	struct sbi_pmu_hw_event *temp;
 	unsigned long mctr_inhbt = 0;
-	u32 hartid = current_hartid();
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 
 	if (cbase >= num_hw_ctrs)
@@ -575,10 +694,10 @@ static int pmu_ctr_find_hw(unsigned long cbase, unsigned long cmask, unsigned lo
 	 * If Sscof is present try to find the programmable counter for
 	 * cycle/instret as well.
 	 */
-	fixed_ctr = pmu_ctr_find_fixed_fw(event_idx);
+	fixed_ctr = pmu_ctr_find_fixed_hw(event_idx);
 	if (fixed_ctr >= 0 &&
 	    !sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
-		return fixed_ctr;
+		return pmu_fixed_ctr_update_inhibit_bits(fixed_ctr, flags);
 
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_11)
 		mctr_inhbt = csr_read(CSR_MCOUNTINHIBIT);
@@ -604,7 +723,7 @@ static int pmu_ctr_find_hw(unsigned long cbase, unsigned long cmask, unsigned lo
 			 * Some of the platform may not support mcountinhibit.
 			 * Checking the active_events is enough for them
 			 */
-			if (active_events[hartid][cbase] != SBI_PMU_EVENT_IDX_INVALID)
+			if (phs->active_events[cbase] != SBI_PMU_EVENT_IDX_INVALID)
 				continue;
 			/* If mcountinhibit is supported, the bit must be enabled */
 			if ((sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_11) &&
@@ -621,7 +740,7 @@ static int pmu_ctr_find_hw(unsigned long cbase, unsigned long cmask, unsigned lo
 		 * Return the fixed counter as they are mandatory anyways.
 		 */
 		if (fixed_ctr >= 0)
-			return fixed_ctr;
+			return pmu_fixed_ctr_update_inhibit_bits(fixed_ctr, flags);
 		else
 			return SBI_EFAIL;
 	}
@@ -639,21 +758,28 @@ static int pmu_ctr_find_hw(unsigned long cbase, unsigned long cmask, unsigned lo
  * Thus, select the first available fw counter after sanity
  * check.
  */
-static int pmu_ctr_find_fw(unsigned long cbase, unsigned long cmask,
-			   uint32_t event_code, u32 hartid)
+static int pmu_ctr_find_fw(struct sbi_pmu_hart_state *phs,
+			   unsigned long cbase, unsigned long cmask,
+			   uint32_t event_code, uint64_t edata)
 {
 	int i, cidx;
 
+	if ((event_code >= SBI_PMU_FW_MAX &&
+	    event_code <= SBI_PMU_FW_RESERVED_MAX) ||
+	    event_code > SBI_PMU_FW_PLATFORM)
+		return SBI_EINVAL;
+
 	for_each_set_bit(i, &cmask, BITS_PER_LONG) {
 		cidx = i + cbase;
 		if (cidx < num_hw_ctrs || total_ctrs <= cidx)
 			continue;
-		if (active_events[hartid][i] != SBI_PMU_EVENT_IDX_INVALID)
+		if (phs->active_events[i] != SBI_PMU_EVENT_IDX_INVALID)
 			continue;
-		if (SBI_PMU_FW_MAX <= event_code &&
-		    pmu_dev && pmu_dev->fw_counter_match_code) {
-			if (!pmu_dev->fw_counter_match_code(cidx - num_hw_ctrs,
-							    event_code))
+		if (SBI_PMU_FW_PLATFORM == event_code &&
+		    pmu_dev && pmu_dev->fw_counter_match_encoding) {
+			if (!pmu_dev->fw_counter_match_encoding(phs->hartid,
+							    cidx - num_hw_ctrs,
+							    edata))
 				continue;
 		}
 
@@ -667,15 +793,15 @@ int sbi_pmu_ctr_cfg_match(unsigned long cidx_base, unsigned long cidx_mask,
 			  unsigned long flags, unsigned long event_idx,
 			  uint64_t event_data)
 {
-	int ret, ctr_idx = SBI_ENOTSUPP;
-	u32 event_code, hartid = current_hartid();
-	int event_type;
+	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
+	int ret, event_type, ctr_idx = SBI_ENOTSUPP;
+	u32 event_code;
 
 	/* Do a basic sanity check of counter base & mask */
 	if ((cidx_base + sbi_fls(cidx_mask)) >= total_ctrs)
 		return SBI_EINVAL;
 
-	event_type = pmu_event_validate(event_idx);
+	event_type = pmu_event_validate(phs, event_idx, event_data);
 	if (event_type < 0)
 		return SBI_EINVAL;
 	event_code = get_cidx_code(event_idx);
@@ -684,25 +810,34 @@ int sbi_pmu_ctr_cfg_match(unsigned long cidx_base, unsigned long cidx_mask,
 		/* The caller wants to skip the match because it already knows the
 		 * counter idx for the given event. Verify that the counter idx
 		 * is still valid.
+		 * As per the specification, we should "unconditionally select
+		 * the first counter from the set of counters specified by the
+		 * counter_idx_base and counter_idx_mask".
 		 */
-		if (active_events[hartid][cidx_base] == SBI_PMU_EVENT_IDX_INVALID)
+		unsigned long cidx_first = cidx_base + sbi_ffs(cidx_mask);
+
+		if (phs->active_events[cidx_first] == SBI_PMU_EVENT_IDX_INVALID)
 			return SBI_EINVAL;
-		ctr_idx = cidx_base;
+		ctr_idx = cidx_first;
 		goto skip_match;
 	}
 
 	if (event_type == SBI_PMU_EVENT_TYPE_FW) {
 		/* Any firmware counter can be used track any firmware event */
-		ctr_idx = pmu_ctr_find_fw(cidx_base, cidx_mask, event_code, hartid);
+		ctr_idx = pmu_ctr_find_fw(phs, cidx_base, cidx_mask,
+					  event_code, event_data);
+		if (event_code == SBI_PMU_FW_PLATFORM)
+			phs->fw_counters_data[ctr_idx - num_hw_ctrs] =
+								event_data;
 	} else {
-		ctr_idx = pmu_ctr_find_hw(cidx_base, cidx_mask, flags, event_idx,
-					  event_data);
+		ctr_idx = pmu_ctr_find_hw(phs, cidx_base, cidx_mask, flags,
+					  event_idx, event_data);
 	}
 
 	if (ctr_idx < 0)
 		return SBI_ENOTSUPP;
 
-	active_events[hartid][ctr_idx] = event_idx;
+	phs->active_events[ctr_idx] = event_idx;
 skip_match:
 	if (event_type == SBI_PMU_EVENT_TYPE_HW) {
 		if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE)
@@ -711,18 +846,17 @@ skip_match:
 			pmu_ctr_start_hw(ctr_idx, 0, false);
 	} else if (event_type == SBI_PMU_EVENT_TYPE_FW) {
 		if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE)
-			fw_counters_value[hartid][ctr_idx - num_hw_ctrs] = 0;
+			phs->fw_counters_data[ctr_idx - num_hw_ctrs] = 0;
 		if (flags & SBI_PMU_CFG_FLAG_AUTO_START) {
-			if (SBI_PMU_FW_MAX <= event_code &&
+			if (SBI_PMU_FW_PLATFORM == event_code &&
 			    pmu_dev && pmu_dev->fw_counter_start) {
 				ret = pmu_dev->fw_counter_start(
-					ctr_idx - num_hw_ctrs, event_code,
-					fw_counters_value[hartid][ctr_idx - num_hw_ctrs],
-					true);
+					phs->hartid,
+					ctr_idx - num_hw_ctrs, event_data);
 				if (ret)
 					return ret;
 			}
-			fw_counters_started[hartid] |= BIT(ctr_idx - num_hw_ctrs);
+			phs->fw_counters_started |= BIT(ctr_idx - num_hw_ctrs);
 		}
 	}
 
@@ -731,19 +865,20 @@ skip_match:
 
 int sbi_pmu_ctr_incr_fw(enum sbi_pmu_fw_event_code_id fw_id)
 {
-	u32 cidx, hartid = current_hartid();
+	u32 cidx;
 	uint64_t *fcounter = NULL;
+	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
 
-	if (likely(!fw_counters_started[hartid]))
+	if (likely(!phs->fw_counters_started))
 		return 0;
 
 	if (unlikely(fw_id >= SBI_PMU_FW_MAX))
 		return SBI_EINVAL;
 
 	for (cidx = num_hw_ctrs; cidx < total_ctrs; cidx++) {
-		if (get_cidx_code(active_events[hartid][cidx]) == fw_id &&
-		    (fw_counters_started[hartid] & BIT(cidx - num_hw_ctrs))) {
-			fcounter = &fw_counters_value[hartid][cidx - num_hw_ctrs];
+		if (get_cidx_code(phs->active_events[cidx]) == fw_id &&
+		    (phs->fw_counters_started & BIT(cidx - num_hw_ctrs))) {
+			fcounter = &phs->fw_counters_data[cidx - num_hw_ctrs];
 			break;
 		}
 	}
@@ -761,15 +896,20 @@ unsigned long sbi_pmu_num_ctr(void)
 
 int sbi_pmu_ctr_get_info(uint32_t cidx, unsigned long *ctr_info)
 {
+	int width;
 	union sbi_pmu_ctr_info cinfo = {0};
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	unsigned long counter_mask = (unsigned long)sbi_hart_mhpm_mask(scratch) |
+				     SBI_PMU_CY_IR_MASK;
 
-	/* Sanity check. Counter1 is not mapped at all */
-	if (cidx >= total_ctrs || cidx == 1)
+	/* Sanity check */
+	if (cidx >= total_ctrs)
 		return SBI_EINVAL;
 
 	/* We have 31 HW counters with 31 being the last index(MHPMCOUNTER31) */
 	if (cidx < num_hw_ctrs) {
+		if (!(__test_bit(cidx, &counter_mask)))
+			return SBI_EINVAL;
 		cinfo.type = SBI_PMU_CTR_TYPE_HW;
 		cinfo.csr = CSR_CYCLE + cidx;
 		/* mcycle & minstret are always 64 bit */
@@ -782,6 +922,11 @@ int sbi_pmu_ctr_get_info(uint32_t cidx, unsigned long *ctr_info)
 		cinfo.type = SBI_PMU_CTR_TYPE_FW;
 		/* Firmware counters are always 64 bits wide */
 		cinfo.width = 63;
+		if (pmu_dev && pmu_dev->fw_counter_width) {
+			width = pmu_dev->fw_counter_width();
+			if (width)
+				cinfo.width = width - 1;
+		}
 	}
 
 	*ctr_info = cinfo.value;
@@ -789,16 +934,16 @@ int sbi_pmu_ctr_get_info(uint32_t cidx, unsigned long *ctr_info)
 	return 0;
 }
 
-static void pmu_reset_event_map(u32 hartid)
+static void pmu_reset_event_map(struct sbi_pmu_hart_state *phs)
 {
 	int j;
 
 	/* Initialize the counter to event mapping table */
 	for (j = 3; j < total_ctrs; j++)
-		active_events[hartid][j] = SBI_PMU_EVENT_IDX_INVALID;
+		phs->active_events[j] = SBI_PMU_EVENT_IDX_INVALID;
 	for (j = 0; j < SBI_PMU_FW_CTR_MAX; j++)
-		fw_counters_value[hartid][j] = 0;
-	fw_counters_started[hartid] = 0;
+		phs->fw_counters_data[j] = 0;
+	phs->fw_counters_started = 0;
 }
 
 const struct sbi_pmu_device *sbi_pmu_get_device(void)
@@ -816,39 +961,71 @@ void sbi_pmu_set_device(const struct sbi_pmu_device *dev)
 
 void sbi_pmu_exit(struct sbi_scratch *scratch)
 {
-	u32 hartid = current_hartid();
-
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_11)
 		csr_write(CSR_MCOUNTINHIBIT, 0xFFFFFFF8);
 
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_10)
 		csr_write(CSR_MCOUNTEREN, -1);
-	pmu_reset_event_map(hartid);
+
+	pmu_reset_event_map(pmu_get_hart_state_ptr(scratch));
 }
 
 int sbi_pmu_init(struct sbi_scratch *scratch, bool cold_boot)
 {
+	int hpm_count = sbi_fls(sbi_hart_mhpm_mask(scratch));
+	struct sbi_pmu_hart_state *phs;
 	const struct sbi_platform *plat;
-	u32 hartid = current_hartid();
+	int rc;
 
 	if (cold_boot) {
+		hw_event_map = sbi_calloc(sizeof(*hw_event_map),
+					  SBI_PMU_HW_EVENT_MAX);
+		if (!hw_event_map)
+			return SBI_ENOMEM;
+
+		phs_ptr_offset = sbi_scratch_alloc_type_offset(void *);
+		if (!phs_ptr_offset) {
+			sbi_free(hw_event_map);
+			return SBI_ENOMEM;
+		}
+
 		plat = sbi_platform_ptr(scratch);
 		/* Initialize hw pmu events */
-		sbi_platform_pmu_init(plat);
+		rc = sbi_platform_pmu_init(plat);
+		if (rc)
+			sbi_dprintf("%s: platform pmu init failed "
+				    "(error %d)\n", __func__, rc);
 
 		/* mcycle & minstret is available always */
-		num_hw_ctrs = sbi_hart_mhpm_count(scratch) + 3;
+		if (!hpm_count)
+			/* Only CY, TM & IR are implemented in the hw */
+			num_hw_ctrs = 3;
+		else
+			num_hw_ctrs = hpm_count + 1;
+
+		if (num_hw_ctrs > SBI_PMU_HW_CTR_MAX)
+			return SBI_EINVAL;
+
 		total_ctrs = num_hw_ctrs + SBI_PMU_FW_CTR_MAX;
 	}
 
-	pmu_reset_event_map(hartid);
+	phs = pmu_get_hart_state_ptr(scratch);
+	if (!phs) {
+		phs = sbi_zalloc(sizeof(*phs));
+		if (!phs)
+			return SBI_ENOMEM;
+		phs->hartid = current_hartid();
+		pmu_set_hart_state_ptr(scratch, phs);
+	}
+
+	pmu_reset_event_map(phs);
 
 	/* First three counters are fixed by the priv spec and we enable it by default */
-	active_events[hartid][0] = SBI_PMU_EVENT_TYPE_HW << SBI_PMU_EVENT_IDX_OFFSET |
-				   SBI_PMU_HW_CPU_CYCLES;
-	active_events[hartid][1] = SBI_PMU_EVENT_IDX_INVALID;
-	active_events[hartid][2] = SBI_PMU_EVENT_TYPE_HW << SBI_PMU_EVENT_IDX_OFFSET |
-				   SBI_PMU_HW_INSTRUCTIONS;
+	phs->active_events[0] = (SBI_PMU_EVENT_TYPE_HW << SBI_PMU_EVENT_IDX_TYPE_OFFSET) |
+				SBI_PMU_HW_CPU_CYCLES;
+	phs->active_events[1] = SBI_PMU_EVENT_IDX_INVALID;
+	phs->active_events[2] = (SBI_PMU_EVENT_TYPE_HW << SBI_PMU_EVENT_IDX_TYPE_OFFSET) |
+				SBI_PMU_HW_INSTRUCTIONS;
 
 	return 0;
 }

lib/sbi/sbi_scratch.c

@@ -14,29 +14,40 @@
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
 
-u32 last_hartid_having_scratch = SBI_HARTMASK_MAX_BITS - 1;
-struct sbi_scratch *hartid_to_scratch_table[SBI_HARTMASK_MAX_BITS] = { 0 };
+u32 last_hartindex_having_scratch = 0;
+u32 hartindex_to_hartid_table[SBI_HARTMASK_MAX_BITS + 1] = { -1U };
+struct sbi_scratch *hartindex_to_scratch_table[SBI_HARTMASK_MAX_BITS + 1] = { 0 };
 
 static spinlock_t extra_lock = SPIN_LOCK_INITIALIZER;
 static unsigned long extra_offset = SBI_SCRATCH_EXTRA_SPACE_OFFSET;
 
+u32 sbi_hartid_to_hartindex(u32 hartid)
+{
+	u32 i;
+
+	for (i = 0; i <= last_hartindex_having_scratch; i++)
+		if (hartindex_to_hartid_table[i] == hartid)
+			return i;
+
+	return -1U;
+}
+
 typedef struct sbi_scratch *(*hartid2scratch)(ulong hartid, ulong hartindex);
 
 int sbi_scratch_init(struct sbi_scratch *scratch)
 {
-	u32 i;
+	u32 i, h;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	for (i = 0; i < SBI_HARTMASK_MAX_BITS; i++) {
-		if (sbi_platform_hart_invalid(plat, i))
-			continue;
-		hartid_to_scratch_table[i] =
-			((hartid2scratch)scratch->hartid_to_scratch)(i,
-					sbi_platform_hart_index(plat, i));
-		if (hartid_to_scratch_table[i])
-			last_hartid_having_scratch = i;
+	for (i = 0; i < plat->hart_count; i++) {
+		h = (plat->hart_index2id) ? plat->hart_index2id[i] : i;
+		hartindex_to_hartid_table[i] = h;
+		hartindex_to_scratch_table[i] =
+			((hartid2scratch)scratch->hartid_to_scratch)(h, i);
 	}
 
+	last_hartindex_having_scratch = plat->hart_count - 1;
+
 	return 0;
 }
 
@@ -59,8 +70,8 @@ unsigned long sbi_scratch_alloc_offset(unsigned long size)
 	if (!size)
 		return 0;
 
-	if (size & (__SIZEOF_POINTER__ - 1))
-		size = (size & ~(__SIZEOF_POINTER__ - 1)) + __SIZEOF_POINTER__;
+	size += __SIZEOF_POINTER__ - 1;
+	size &= ~((unsigned long)__SIZEOF_POINTER__ - 1);
 
 	spin_lock(&extra_lock);
 
@@ -74,8 +85,8 @@ done:
 	spin_unlock(&extra_lock);
 
 	if (ret) {
-		for (i = 0; i <= sbi_scratch_last_hartid(); i++) {
-			rscratch = sbi_hartid_to_scratch(i);
+		for (i = 0; i <= sbi_scratch_last_hartindex(); i++) {
+			rscratch = sbi_hartindex_to_scratch(i);
 			if (!rscratch)
 				continue;
 			ptr = sbi_scratch_offset_ptr(rscratch, ret);
@@ -97,3 +108,14 @@ void sbi_scratch_free_offset(unsigned long offset)
 	 * brain-dead allocator.
 	 */
 }
+
+unsigned long sbi_scratch_used_space(void)
+{
+	unsigned long ret = 0;
+
+	spin_lock(&extra_lock);
+	ret = extra_offset;
+	spin_unlock(&extra_lock);
+
+	return ret;
+}

lib/sbi/sbi_system.c

@@ -17,6 +17,7 @@
 #include <sbi/sbi_system.h>
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_init.h>
+#include <sbi/sbi_timer.h>
 
 static SBI_LIST_HEAD(reset_devices_list);
 
@@ -71,7 +72,8 @@ void __noreturn sbi_system_reset(u32 reset_type, u32 reset_reason)
 
 	/* Send HALT IPI to every hart other than the current hart */
 	while (!sbi_hsm_hart_interruptible_mask(dom, hbase, &hmask)) {
-		if (hbase <= cur_hartid)
+		if ((hbase <= cur_hartid)
+			  && (cur_hartid < hbase + BITS_PER_LONG))
 			hmask &= ~(1UL << (cur_hartid - hbase));
 		if (hmask)
 			sbi_ipi_send_halt(hmask, hbase);
@@ -79,7 +81,7 @@ void __noreturn sbi_system_reset(u32 reset_type, u32 reset_reason)
 	}
 
 	/* Stop current HART */
-	sbi_hsm_hart_stop(scratch, FALSE);
+	sbi_hsm_hart_stop(scratch, false);
 
 	/* Platform specific reset if domain allowed system reset */
 	if (dom->system_reset_allowed) {
@@ -92,3 +94,117 @@ void __noreturn sbi_system_reset(u32 reset_type, u32 reset_reason)
 	/* If platform specific reset did not work then do sbi_exit() */
 	sbi_exit(scratch);
 }
+
+static const struct sbi_system_suspend_device *suspend_dev = NULL;
+
+const struct sbi_system_suspend_device *sbi_system_suspend_get_device(void)
+{
+	return suspend_dev;
+}
+
+void sbi_system_suspend_set_device(struct sbi_system_suspend_device *dev)
+{
+	if (!dev || suspend_dev)
+		return;
+
+	suspend_dev = dev;
+}
+
+static int sbi_system_suspend_test_check(u32 sleep_type)
+{
+	return sleep_type == SBI_SUSP_SLEEP_TYPE_SUSPEND ? 0 : SBI_EINVAL;
+}
+
+static int sbi_system_suspend_test_suspend(u32 sleep_type,
+					   unsigned long mmode_resume_addr)
+{
+	if (sleep_type != SBI_SUSP_SLEEP_TYPE_SUSPEND)
+		return SBI_EINVAL;
+
+	sbi_timer_mdelay(5000);
+
+	/* Wait for interrupt */
+	wfi();
+
+	return SBI_OK;
+}
+
+static struct sbi_system_suspend_device sbi_system_suspend_test = {
+	.name = "system-suspend-test",
+	.system_suspend_check = sbi_system_suspend_test_check,
+	.system_suspend = sbi_system_suspend_test_suspend,
+};
+
+void sbi_system_suspend_test_enable(void)
+{
+	sbi_system_suspend_set_device(&sbi_system_suspend_test);
+}
+
+bool sbi_system_suspend_supported(u32 sleep_type)
+{
+	return suspend_dev && suspend_dev->system_suspend_check &&
+	       suspend_dev->system_suspend_check(sleep_type) == 0;
+}
+
+int sbi_system_suspend(u32 sleep_type, ulong resume_addr, ulong opaque)
+{
+	const struct sbi_domain *dom = sbi_domain_thishart_ptr();
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	void (*jump_warmboot)(void) = (void (*)(void))scratch->warmboot_addr;
+	unsigned int hartid = current_hartid();
+	unsigned long prev_mode;
+	unsigned long i, j;
+	int ret;
+
+	if (!dom || !dom->system_suspend_allowed)
+		return SBI_EFAIL;
+
+	if (!suspend_dev || !suspend_dev->system_suspend ||
+	    !suspend_dev->system_suspend_check)
+		return SBI_EFAIL;
+
+	ret = suspend_dev->system_suspend_check(sleep_type);
+	if (ret != SBI_OK)
+		return ret;
+
+	prev_mode = (csr_read(CSR_MSTATUS) & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT;
+	if (prev_mode != PRV_S && prev_mode != PRV_U)
+		return SBI_EFAIL;
+
+	sbi_hartmask_for_each_hartindex(j, &dom->assigned_harts) {
+		i = sbi_hartindex_to_hartid(j);
+		if (i == hartid)
+			continue;
+		if (__sbi_hsm_hart_get_state(i) != SBI_HSM_STATE_STOPPED)
+			return SBI_ERR_DENIED;
+	}
+
+	if (!sbi_domain_check_addr(dom, resume_addr, prev_mode,
+				   SBI_DOMAIN_EXECUTE))
+		return SBI_EINVALID_ADDR;
+
+	if (!sbi_hsm_hart_change_state(scratch, SBI_HSM_STATE_STARTED,
+				       SBI_HSM_STATE_SUSPENDED))
+		return SBI_EFAIL;
+
+	/* Prepare for resume */
+	scratch->next_mode = prev_mode;
+	scratch->next_addr = resume_addr;
+	scratch->next_arg1 = opaque;
+
+	__sbi_hsm_suspend_non_ret_save(scratch);
+
+	/* Suspend */
+	ret = suspend_dev->system_suspend(sleep_type, scratch->warmboot_addr);
+	if (ret != SBI_OK) {
+		if (!sbi_hsm_hart_change_state(scratch, SBI_HSM_STATE_SUSPENDED,
+					       SBI_HSM_STATE_STARTED))
+			sbi_hart_hang();
+		return ret;
+	}
+
+	/* Resume */
+	jump_warmboot();
+
+	__builtin_unreachable();
+}

lib/sbi/sbi_timer.c

@@ -188,7 +188,7 @@ int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot)
 		if (!time_delta_off)
 			return SBI_ENOMEM;
 
-		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_TIME))
+		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_ZICNTR))
 			get_time_val = get_ticks;
 	} else {
 		if (!time_delta_off)

lib/sbi/sbi_tlb.c

@@ -14,6 +14,7 @@
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_fifo.h>
 #include <sbi/sbi_hart.h>
+#include <sbi/sbi_heap.h>
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_tlb.h>
@@ -33,7 +34,7 @@ static void tlb_flush_all(void)
 	__asm__ __volatile("sfence.vma");
 }
 
-void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -58,7 +59,7 @@ done:
 	csr_write(CSR_HGATP, hgatp);
 }
 
-void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -76,7 +77,7 @@ void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo)
 	}
 }
 
-void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -97,7 +98,7 @@ void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo)
 	}
 }
 
-void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -110,12 +111,7 @@ void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
 	hgatp = csr_swap(CSR_HGATP,
 			 (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK);
 
-	if (start == 0 && size == 0) {
-		__sbi_hfence_vvma_all();
-		goto done;
-	}
-
-	if (size == SBI_TLB_FLUSH_ALL) {
+	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
 		__sbi_hfence_vvma_asid(asid);
 		goto done;
 	}
@@ -128,7 +124,7 @@ done:
 	csr_write(CSR_HGATP, hgatp);
 }
 
-void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -137,12 +133,7 @@ void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
 
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_GVMA_VMID_RCVD);
 
-	if (start == 0 && size == 0) {
-		__sbi_hfence_gvma_all();
-		return;
-	}
-
-	if (size == SBI_TLB_FLUSH_ALL) {
+	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
 		__sbi_hfence_gvma_vmid(vmid);
 		return;
 	}
@@ -152,7 +143,7 @@ void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
 	}
 }
 
-void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -161,13 +152,8 @@ void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
 
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SFENCE_VMA_ASID_RCVD);
 
-	if (start == 0 && size == 0) {
-		tlb_flush_all();
-		return;
-	}
-
 	/* Flush entire MM context for a given ASID */
-	if (size == SBI_TLB_FLUSH_ALL) {
+	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
 		__asm__ __volatile__("sfence.vma x0, %0"
 				     :
 				     : "r"(asid)
@@ -183,89 +169,93 @@ void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
 	}
 }
 
-void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo)
 {
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_FENCE_I_RECVD);
 
 	__asm__ __volatile("fence.i");
 }
 
-static void tlb_pmu_incr_fw_ctr(struct sbi_tlb_info *data)
+static void tlb_entry_local_process(struct sbi_tlb_info *data)
 {
 	if (unlikely(!data))
 		return;
 
-	if (data->local_fn == sbi_tlb_local_fence_i)
-		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_FENCE_I_SENT);
-	else if (data->local_fn == sbi_tlb_local_sfence_vma)
-		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SFENCE_VMA_SENT);
-	else if (data->local_fn == sbi_tlb_local_sfence_vma_asid)
-		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_SFENCE_VMA_ASID_SENT);
-	else if (data->local_fn == sbi_tlb_local_hfence_gvma)
-		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_GVMA_SENT);
-	else if (data->local_fn == sbi_tlb_local_hfence_gvma_vmid)
-		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_GVMA_VMID_SENT);
-	else if (data->local_fn == sbi_tlb_local_hfence_vvma)
-		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_VVMA_SENT);
-	else if (data->local_fn == sbi_tlb_local_hfence_vvma_asid)
-		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_VVMA_ASID_SENT);
+	switch (data->type) {
+	case SBI_TLB_FENCE_I:
+		sbi_tlb_local_fence_i(data);
+		break;
+	case SBI_TLB_SFENCE_VMA:
+		sbi_tlb_local_sfence_vma(data);
+		break;
+	case SBI_TLB_SFENCE_VMA_ASID:
+		sbi_tlb_local_sfence_vma_asid(data);
+		break;
+	case SBI_TLB_HFENCE_GVMA_VMID:
+		sbi_tlb_local_hfence_gvma_vmid(data);
+		break;
+	case SBI_TLB_HFENCE_GVMA:
+		sbi_tlb_local_hfence_gvma(data);
+		break;
+	case SBI_TLB_HFENCE_VVMA_ASID:
+		sbi_tlb_local_hfence_vvma_asid(data);
+		break;
+	case SBI_TLB_HFENCE_VVMA:
+		sbi_tlb_local_hfence_vvma(data);
+		break;
+	default:
+		break;
+	};
 }
 
 static void tlb_entry_process(struct sbi_tlb_info *tinfo)
 {
-	u32 rhartid;
+	u32 rindex;
 	struct sbi_scratch *rscratch = NULL;
-	unsigned long *rtlb_sync = NULL;
+	atomic_t *rtlb_sync = NULL;
 
-	tinfo->local_fn(tinfo);
+	tlb_entry_local_process(tinfo);
 
-	sbi_hartmask_for_each_hart(rhartid, &tinfo->smask) {
-		rscratch = sbi_hartid_to_scratch(rhartid);
+	sbi_hartmask_for_each_hartindex(rindex, &tinfo->smask) {
+		rscratch = sbi_hartindex_to_scratch(rindex);
 		if (!rscratch)
 			continue;
 
 		rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off);
-		while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ;
+		atomic_sub_return(rtlb_sync, 1);
 	}
 }
 
-static void tlb_process_count(struct sbi_scratch *scratch, int count)
+static bool tlb_process_once(struct sbi_scratch *scratch)
 {
 	struct sbi_tlb_info tinfo;
-	unsigned int deq_count = 0;
 	struct sbi_fifo *tlb_fifo =
 			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
 
-	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
+	if (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
 		tlb_entry_process(&tinfo);
-		deq_count++;
-		if (deq_count > count)
-			break;
-
+		return true;
 	}
+
+	return false;
 }
 
 static void tlb_process(struct sbi_scratch *scratch)
 {
-	struct sbi_tlb_info tinfo;
-	struct sbi_fifo *tlb_fifo =
-			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
-
-	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo))
-		tlb_entry_process(&tinfo);
+	while (tlb_process_once(scratch));
 }
 
 static void tlb_sync(struct sbi_scratch *scratch)
 {
-	unsigned long *tlb_sync =
+	atomic_t *tlb_sync =
 			sbi_scratch_offset_ptr(scratch, tlb_sync_off);
 
-	while (!atomic_raw_xchg_ulong(tlb_sync, 0)) {
+	while (atomic_read(tlb_sync) > 0) {
 		/*
 		 * While we are waiting for remote hart to set the sync,
 		 * consume fifo requests to avoid deadlock.
 		 */
-		tlb_process_count(scratch, 1);
+		tlb_process_once(scratch);
 	}
 
 	return;
@@ -326,12 +316,12 @@ static int tlb_update_cb(void *in, void *data)
 	curr = (struct sbi_tlb_info *)data;
 	next = (struct sbi_tlb_info *)in;
 
-	if (next->local_fn == sbi_tlb_local_sfence_vma_asid &&
-	    curr->local_fn == sbi_tlb_local_sfence_vma_asid) {
+	if (next->type == SBI_TLB_SFENCE_VMA_ASID &&
+	    curr->type == SBI_TLB_SFENCE_VMA_ASID) {
 		if (next->asid == curr->asid)
 			ret = tlb_range_check(curr, next);
-	} else if (next->local_fn == sbi_tlb_local_sfence_vma &&
-		   curr->local_fn == sbi_tlb_local_sfence_vma) {
+	} else if (next->type == SBI_TLB_SFENCE_VMA &&
+		   curr->type == SBI_TLB_SFENCE_VMA) {
 		ret = tlb_range_check(curr, next);
 	}
 
@@ -340,40 +330,28 @@ static int tlb_update_cb(void *in, void *data)
 
 static int tlb_update(struct sbi_scratch *scratch,
 			  struct sbi_scratch *remote_scratch,
-			  u32 remote_hartid, void *data)
+			  u32 remote_hartindex, void *data)
 {
 	int ret;
+	atomic_t *tlb_sync;
 	struct sbi_fifo *tlb_fifo_r;
 	struct sbi_tlb_info *tinfo = data;
 	u32 curr_hartid = current_hartid();
 
-	/*
-	 * If address range to flush is too big then simply
-	 * upgrade it to flush all because we can only flush
-	 * 4KB at a time.
-	 */
-	if (tinfo->size > tlb_range_flush_limit) {
-		tinfo->start = 0;
-		tinfo->size = SBI_TLB_FLUSH_ALL;
-	}
-
 	/*
 	 * If the request is to queue a tlb flush entry for itself
 	 * then just do a local flush and return;
 	 */
-	if (remote_hartid == curr_hartid) {
-		tinfo->local_fn(tinfo);
-		return -1;
+	if (sbi_hartindex_to_hartid(remote_hartindex) == curr_hartid) {
+		tlb_entry_local_process(tinfo);
+		return SBI_IPI_UPDATE_BREAK;
 	}
 
 	tlb_fifo_r = sbi_scratch_offset_ptr(remote_scratch, tlb_fifo_off);
 
 	ret = sbi_fifo_inplace_update(tlb_fifo_r, data, tlb_update_cb);
-	if (ret != SBI_FIFO_UNCHANGED) {
-		return 1;
-	}
 
-	while (sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
+	if (ret == SBI_FIFO_UNCHANGED && sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
 		/**
 		 * For now, Busy loop until there is space in the fifo.
 		 * There may be case where target hart is also
@@ -382,12 +360,16 @@ static int tlb_update(struct sbi_scratch *scratch,
 		 * TODO: Introduce a wait/wakeup event mechanism to handle
 		 * this properly.
 		 */
-		tlb_process_count(scratch, 1);
-		sbi_dprintf("hart%d: hart%d tlb fifo full\n",
-			    curr_hartid, remote_hartid);
+		tlb_process_once(scratch);
+		sbi_dprintf("hart%d: hart%d tlb fifo full\n", curr_hartid,
+			    sbi_hartindex_to_hartid(remote_hartindex));
+		return SBI_IPI_UPDATE_RETRY;
 	}
 
-	return 0;
+	tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);
+	atomic_add_return(tlb_sync, 1);
+
+	return SBI_IPI_UPDATE_SUCCESS;
 }
 
 static struct sbi_ipi_event_ops tlb_ops = {
@@ -399,12 +381,32 @@ static struct sbi_ipi_event_ops tlb_ops = {
 
 static u32 tlb_event = SBI_IPI_EVENT_MAX;
 
+static const u32 tlb_type_to_pmu_fw_event[SBI_TLB_TYPE_MAX] = {
+	[SBI_TLB_FENCE_I] = SBI_PMU_FW_FENCE_I_SENT,
+	[SBI_TLB_SFENCE_VMA] = SBI_PMU_FW_SFENCE_VMA_SENT,
+	[SBI_TLB_SFENCE_VMA_ASID] = SBI_PMU_FW_SFENCE_VMA_ASID_SENT,
+	[SBI_TLB_HFENCE_GVMA_VMID] = SBI_PMU_FW_HFENCE_GVMA_VMID_SENT,
+	[SBI_TLB_HFENCE_GVMA] = SBI_PMU_FW_HFENCE_GVMA_SENT,
+	[SBI_TLB_HFENCE_VVMA_ASID] = SBI_PMU_FW_HFENCE_VVMA_ASID_SENT,
+	[SBI_TLB_HFENCE_VVMA] = SBI_PMU_FW_HFENCE_VVMA_SENT,
+};
+
 int sbi_tlb_request(ulong hmask, ulong hbase, struct sbi_tlb_info *tinfo)
 {
-	if (!tinfo->local_fn)
+	if (tinfo->type < 0 || tinfo->type >= SBI_TLB_TYPE_MAX)
 		return SBI_EINVAL;
 
-	tlb_pmu_incr_fw_ctr(tinfo);
+	/*
+	 * If address range to flush is too big then simply
+	 * upgrade it to flush all because we can only flush
+	 * 4KB at a time.
+	 */
+	if (tinfo->size > tlb_range_flush_limit) {
+		tinfo->start = 0;
+		tinfo->size = SBI_TLB_FLUSH_ALL;
+	}
+
+	sbi_pmu_ctr_incr_fw(tlb_type_to_pmu_fw_event[tinfo->type]);
 
 	return sbi_ipi_send_many(hmask, hbase, tlb_event, tinfo);
 }
@@ -413,7 +415,7 @@ int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
 {
 	int ret;
 	void *tlb_mem;
-	unsigned long *tlb_sync;
+	atomic_t *tlb_sync;
 	struct sbi_fifo *tlb_q;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
@@ -426,8 +428,7 @@ int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
 			sbi_scratch_free_offset(tlb_sync_off);
 			return SBI_ENOMEM;
 		}
-		tlb_fifo_mem_off = sbi_scratch_alloc_offset(
-				SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE);
+		tlb_fifo_mem_off = sbi_scratch_alloc_offset(sizeof(tlb_mem));
 		if (!tlb_fifo_mem_off) {
 			sbi_scratch_free_offset(tlb_fifo_off);
 			sbi_scratch_free_offset(tlb_sync_off);
@@ -453,12 +454,19 @@ int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
 
 	tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);
 	tlb_q = sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
-	tlb_mem = sbi_scratch_offset_ptr(scratch, tlb_fifo_mem_off);
+	tlb_mem = sbi_scratch_read_type(scratch, void *, tlb_fifo_mem_off);
+	if (!tlb_mem) {
+		tlb_mem = sbi_malloc(
+				sbi_platform_tlb_fifo_num_entries(plat) * SBI_TLB_INFO_SIZE);
+		if (!tlb_mem)
+			return SBI_ENOMEM;
+		sbi_scratch_write_type(scratch, void *, tlb_fifo_mem_off, tlb_mem);
+	}
 
-	*tlb_sync = 0;
+	ATOMIC_INIT(tlb_sync, 0);
 
 	sbi_fifo_init(tlb_q, tlb_mem,
-		      SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);
+		      sbi_platform_tlb_fifo_num_entries(plat), SBI_TLB_INFO_SIZE);
 
 	return 0;
 }

lib/sbi/sbi_trap.c

@@ -88,12 +88,12 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs,
 {
 	ulong hstatus, vsstatus, prev_mode;
 #if __riscv_xlen == 32
-	bool prev_virt = (regs->mstatusH & MSTATUSH_MPV) ? TRUE : FALSE;
+	bool prev_virt = (regs->mstatusH & MSTATUSH_MPV) ? true : false;
 #else
-	bool prev_virt = (regs->mstatus & MSTATUS_MPV) ? TRUE : FALSE;
+	bool prev_virt = (regs->mstatus & MSTATUS_MPV) ? true : false;
 #endif
 	/* By default, we redirect to HS-mode */
-	bool next_virt = FALSE;
+	bool next_virt = false;
 
 	/* Sanity check on previous mode */
 	prev_mode = (regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT;
@@ -106,7 +106,7 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs,
 	if (misa_extension('H') && prev_virt) {
 		if ((trap->cause < __riscv_xlen) &&
 		    (csr_read(CSR_HEDELEG) & BIT(trap->cause))) {
-			next_virt = TRUE;
+			next_virt = true;
 		}
 	}
 
@@ -212,7 +212,7 @@ static int sbi_trap_nonaia_irq(struct sbi_trap_regs *regs, ulong mcause)
 		return sbi_irqchip_process(regs);
 	default:
 		return SBI_ENOENT;
-	};
+	}
 
 	return 0;
 }
@@ -320,7 +320,7 @@ struct sbi_trap_regs *sbi_trap_handler(struct sbi_trap_regs *regs)
 
 		rc = sbi_trap_redirect(regs, &trap);
 		break;
-	};
+	}
 
 trap_error:
 	if (rc)

lib/sbi/sbi_unpriv.c

@@ -163,7 +163,7 @@ ulong sbi_get_insn(ulong mepc, struct sbi_trap_info *trap)
 		break;
 	default:
 		break;
-	};
+	}
 
 	return insn;
 }

lib/utils/Kconfig

@@ -14,6 +14,8 @@ source "$(OPENSBI_SRC_DIR)/lib/utils/irqchip/Kconfig"
 
 source "$(OPENSBI_SRC_DIR)/lib/utils/libfdt/Kconfig"
 
+source "$(OPENSBI_SRC_DIR)/lib/utils/regmap/Kconfig"
+
 source "$(OPENSBI_SRC_DIR)/lib/utils/reset/Kconfig"
 
 source "$(OPENSBI_SRC_DIR)/lib/utils/serial/Kconfig"

lib/utils/fdt/Kconfig

@@ -15,4 +15,11 @@ config FDT_PMU
 	bool "FDT performance monitoring unit (PMU) support"
 	default n
 
+config FDT_FIXUPS_PRESERVE_PMU_NODE
+	bool "Preserve PMU node in device-tree"
+	depends on FDT_PMU
+	default n
+	help
+	  Preserve PMU node properties for debugging purposes.
+
 endif

lib/utils/fdt/fdt_domain.c

@@ -13,6 +13,7 @@
 #include <sbi/sbi_domain.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_heap.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi_utils/fdt/fdt_domain.h>
 #include <sbi_utils/fdt/fdt_helper.h>
@@ -219,14 +220,13 @@ skip_device_disable:
 	fdt_nop_node(fdt, poffset);
 }
 
-#define FDT_DOMAIN_MAX_COUNT		8
 #define FDT_DOMAIN_REGION_MAX_COUNT	16
 
-static u32 fdt_domains_count;
-static struct sbi_domain fdt_domains[FDT_DOMAIN_MAX_COUNT];
-static struct sbi_hartmask fdt_masks[FDT_DOMAIN_MAX_COUNT];
-static struct sbi_domain_memregion
-	fdt_regions[FDT_DOMAIN_MAX_COUNT][FDT_DOMAIN_REGION_MAX_COUNT + 1];
+struct parse_region_data {
+	struct sbi_domain *dom;
+	u32 region_count;
+	u32 max_regions;
+};
 
 static int __fdt_parse_region(void *fdt, int domain_offset,
 			      int region_offset, u32 region_access,
@@ -236,13 +236,25 @@ static int __fdt_parse_region(void *fdt, int domain_offset,
 	u32 val32;
 	u64 val64;
 	const u32 *val;
-	u32 *region_count = opaque;
+	struct parse_region_data *preg = opaque;
 	struct sbi_domain_memregion *region;
 
-	/* Find next region of the domain */
-	if (FDT_DOMAIN_REGION_MAX_COUNT <= *region_count)
+	/*
+	 * Non-root domains cannot add a region with only M-mode
+	 * access permissions. M-mode regions can only be part of
+	 * root domain.
+	 *
+	 * SU permission bits can't be all zeroes when M-mode permission
+	 * bits have at least one bit set.
+	 */
+	if (!(region_access & SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK)
+	    && (region_access & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK))
 		return SBI_EINVAL;
-	region = &fdt_regions[fdt_domains_count][*region_count];
+
+	/* Find next region of the domain */
+	if (preg->max_regions <= preg->region_count)
+		return SBI_ENOSPC;
+	region = &preg->dom->regions[preg->region_count];
 
 	/* Read "base" DT property */
 	val = fdt_getprop(fdt, region_offset, "base", &len);
@@ -266,7 +278,7 @@ static int __fdt_parse_region(void *fdt, int domain_offset,
 	if (fdt_get_property(fdt, region_offset, "mmio", NULL))
 		region->flags |= SBI_DOMAIN_MEMREGION_MMIO;
 
-	(*region_count)++;
+	preg->region_count++;
 
 	return 0;
 }
@@ -279,16 +291,30 @@ static int __fdt_parse_domain(void *fdt, int domain_offset, void *opaque)
 	struct sbi_domain *dom;
 	struct sbi_hartmask *mask;
 	struct sbi_hartmask assign_mask;
+	struct parse_region_data preg;
 	int *cold_domain_offset = opaque;
-	struct sbi_domain_memregion *reg, *regions;
-	int i, err, len, cpus_offset, cpu_offset, doffset;
+	struct sbi_domain_memregion *reg;
+	int i, err = 0, len, cpus_offset, cpu_offset, doffset;
 
-	/* Sanity check on maximum domains we can handle */
-	if (FDT_DOMAIN_MAX_COUNT <= fdt_domains_count)
-		return SBI_EINVAL;
-	dom = &fdt_domains[fdt_domains_count];
-	mask = &fdt_masks[fdt_domains_count];
-	regions = &fdt_regions[fdt_domains_count][0];
+	dom = sbi_zalloc(sizeof(*dom));
+	if (!dom)
+		return SBI_ENOMEM;
+
+	dom->regions = sbi_calloc(sizeof(*dom->regions),
+				  FDT_DOMAIN_REGION_MAX_COUNT + 1);
+	if (!dom->regions) {
+		err = SBI_ENOMEM;
+		goto fail_free_domain;
+	}
+	preg.dom = dom;
+	preg.region_count = 0;
+	preg.max_regions = FDT_DOMAIN_REGION_MAX_COUNT;
+
+	mask = sbi_zalloc(sizeof(*mask));
+	if (!mask) {
+		err = SBI_ENOMEM;
+		goto fail_free_regions;
+	}
 
 	/* Read DT node name */
 	strncpy(dom->name, fdt_get_name(fdt, domain_offset, NULL),
@@ -304,29 +330,27 @@ static int __fdt_parse_domain(void *fdt, int domain_offset, void *opaque)
 		for (i = 0; i < len; i++) {
 			cpu_offset = fdt_node_offset_by_phandle(fdt,
 							fdt32_to_cpu(val[i]));
-			if (cpu_offset < 0)
-				return cpu_offset;
+			if (cpu_offset < 0) {
+				err = cpu_offset;
+				goto fail_free_all;
+			}
 
 			err = fdt_parse_hart_id(fdt, cpu_offset, &val32);
 			if (err)
-				return err;
+				goto fail_free_all;
 
 			if (!fdt_node_is_enabled(fdt, cpu_offset))
 				continue;
 
-			sbi_hartmask_set_hart(val32, mask);
+			sbi_hartmask_set_hartid(val32, mask);
 		}
 	}
 
 	/* Setup memregions from DT */
-	val32 = 0;
-	memset(regions, 0,
-		   sizeof(*regions) * (FDT_DOMAIN_REGION_MAX_COUNT + 1));
-	dom->regions = regions;
-	err = fdt_iterate_each_memregion(fdt, domain_offset, &val32,
+	err = fdt_iterate_each_memregion(fdt, domain_offset, &preg,
 					 __fdt_parse_region);
 	if (err)
-		return err;
+		goto fail_free_all;
 
 	/*
 	 * Copy over root domain memregions which don't allow
@@ -338,14 +362,17 @@ static int __fdt_parse_domain(void *fdt, int domain_offset, void *opaque)
 	 * 2) mmio regions protecting M-mode only mmio devices
 	 */
 	sbi_domain_for_each_memregion(&root, reg) {
-		if ((reg->flags & SBI_DOMAIN_MEMREGION_READABLE) ||
-		    (reg->flags & SBI_DOMAIN_MEMREGION_WRITEABLE) ||
-		    (reg->flags & SBI_DOMAIN_MEMREGION_EXECUTABLE))
+		if ((reg->flags & SBI_DOMAIN_MEMREGION_SU_READABLE) ||
+		    (reg->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE) ||
+		    (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE))
 			continue;
-		if (FDT_DOMAIN_REGION_MAX_COUNT <= val32)
-			return SBI_EINVAL;
-		memcpy(&regions[val32++], reg, sizeof(*reg));
+		if (preg.max_regions <= preg.region_count) {
+			err = SBI_EINVAL;
+			goto fail_free_all;
+		}
+		memcpy(&dom->regions[preg.region_count++], reg, sizeof(*reg));
 	}
+	dom->fw_region_inited = root.fw_region_inited;
 
 	/* Read "boot-hart" DT property */
 	val32 = -1U;
@@ -401,14 +428,23 @@ static int __fdt_parse_domain(void *fdt, int domain_offset, void *opaque)
 	/* Read "system-reset-allowed" DT property */
 	if (fdt_get_property(fdt, domain_offset,
 			     "system-reset-allowed", NULL))
-		dom->system_reset_allowed = TRUE;
+		dom->system_reset_allowed = true;
 	else
-		dom->system_reset_allowed = FALSE;
+		dom->system_reset_allowed = false;
+
+	/* Read "system-suspend-allowed" DT property */
+	if (fdt_get_property(fdt, domain_offset,
+			     "system-suspend-allowed", NULL))
+		dom->system_suspend_allowed = true;
+	else
+		dom->system_suspend_allowed = false;
 
 	/* Find /cpus DT node */
 	cpus_offset = fdt_path_offset(fdt, "/cpus");
-	if (cpus_offset < 0)
-		return cpus_offset;
+	if (cpus_offset < 0) {
+		err = cpus_offset;
+		goto fail_free_all;
+	}
 
 	/* HART to domain assignment mask based on CPU DT nodes */
 	sbi_hartmask_clear_all(&assign_mask);
@@ -424,22 +460,35 @@ static int __fdt_parse_domain(void *fdt, int domain_offset, void *opaque)
 			continue;
 
 		val = fdt_getprop(fdt, cpu_offset, "opensbi-domain", &len);
-		if (!val || len < 4)
-			return SBI_EINVAL;
+		if (!val || len < 4) {
+			err = SBI_EINVAL;
+			goto fail_free_all;
+		}
 
 		doffset = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(*val));
-		if (doffset < 0)
-			return doffset;
+		if (doffset < 0) {
+			err = doffset;
+			goto fail_free_all;
+		}
 
 		if (doffset == domain_offset)
-			sbi_hartmask_set_hart(val32, &assign_mask);
+			sbi_hartmask_set_hartid(val32, &assign_mask);
 	}
 
-	/* Increment domains count */
-	fdt_domains_count++;
-
 	/* Register the domain */
-	return sbi_domain_register(dom, &assign_mask);
+	err = sbi_domain_register(dom, &assign_mask);
+	if (err)
+		goto fail_free_all;
+
+	return 0;
+
+fail_free_all:
+	sbi_free(mask);
+fail_free_regions:
+	sbi_free(dom->regions);
+fail_free_domain:
+	sbi_free(dom);
+	return err;
 }
 
 int fdt_domains_populate(void *fdt)

lib/utils/fdt/fdt_fixup.c

@@ -1,3 +1,4 @@
+
 // SPDX-License-Identifier: BSD-2-Clause
 /*
  * fdt_fixup.c - Flat Device Tree parsing helper routines
@@ -14,10 +15,96 @@
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
+#include <sbi/sbi_error.h>
 #include <sbi_utils/fdt/fdt_fixup.h>
 #include <sbi_utils/fdt/fdt_pmu.h>
 #include <sbi_utils/fdt/fdt_helper.h>
 
+int fdt_add_cpu_idle_states(void *fdt, const struct sbi_cpu_idle_state *state)
+{
+	int cpu_node, cpus_node, err, idle_states_node;
+	uint32_t count, phandle;
+
+	err = fdt_open_into(fdt, fdt, fdt_totalsize(fdt) + 1024);
+	if (err < 0)
+		return err;
+
+	err = fdt_find_max_phandle(fdt, &phandle);
+	phandle++;
+	if (err < 0)
+		return err;
+
+	cpus_node = fdt_path_offset(fdt, "/cpus");
+	if (cpus_node < 0)
+		return cpus_node;
+
+	/* Do nothing if the idle-states node already exists. */
+	idle_states_node = fdt_subnode_offset(fdt, cpus_node, "idle-states");
+	if (idle_states_node >= 0)
+		return 0;
+
+	/* Create the idle-states node and its child nodes. */
+	idle_states_node = fdt_add_subnode(fdt, cpus_node, "idle-states");
+	if (idle_states_node < 0)
+		return idle_states_node;
+
+	for (count = 0; state->name; count++, phandle++, state++) {
+		int idle_state_node;
+
+		idle_state_node = fdt_add_subnode(fdt, idle_states_node,
+						  state->name);
+		if (idle_state_node < 0)
+			return idle_state_node;
+
+		fdt_setprop_string(fdt, idle_state_node, "compatible",
+				   "riscv,idle-state");
+		fdt_setprop_u32(fdt, idle_state_node,
+				"riscv,sbi-suspend-param",
+				state->suspend_param);
+		if (state->local_timer_stop)
+			fdt_setprop_empty(fdt, idle_state_node,
+					  "local-timer-stop");
+		fdt_setprop_u32(fdt, idle_state_node, "entry-latency-us",
+				state->entry_latency_us);
+		fdt_setprop_u32(fdt, idle_state_node, "exit-latency-us",
+				state->exit_latency_us);
+		fdt_setprop_u32(fdt, idle_state_node, "min-residency-us",
+				state->min_residency_us);
+		if (state->wakeup_latency_us)
+			fdt_setprop_u32(fdt, idle_state_node,
+					"wakeup-latency-us",
+					state->wakeup_latency_us);
+		fdt_setprop_u32(fdt, idle_state_node, "phandle", phandle);
+	}
+
+	if (count == 0)
+		return 0;
+
+	/* Link each cpu node to the idle state nodes. */
+	fdt_for_each_subnode(cpu_node, fdt, cpus_node) {
+		const char *device_type;
+		fdt32_t *value;
+
+		/* Only process child nodes with device_type = "cpu". */
+		device_type = fdt_getprop(fdt, cpu_node, "device_type", NULL);
+		if (!device_type || strcmp(device_type, "cpu"))
+			continue;
+
+		/* Allocate space for the list of phandles. */
+		err = fdt_setprop_placeholder(fdt, cpu_node, "cpu-idle-states",
+					      count * sizeof(phandle),
+					      (void **)&value);
+		if (err < 0)
+			return err;
+
+		/* Fill in the phandles of the idle state nodes. */
+		for (uint32_t i = 0; i < count; ++i)
+			value[i] = cpu_to_fdt32(phandle - count + i);
+	}
+
+	return 0;
+}
+
 void fdt_cpu_fixup(void *fdt)
 {
 	struct sbi_domain *dom = sbi_domain_thishart_ptr();
@@ -123,7 +210,7 @@ void fdt_plic_fixup(void *fdt)
 
 static int fdt_resv_memory_update_node(void *fdt, unsigned long addr,
 				       unsigned long size, int index,
-				       int parent, bool no_map)
+				       int parent)
 {
 	int na = fdt_address_cells(fdt, 0);
 	int ns = fdt_size_cells(fdt, 0);
@@ -152,16 +239,14 @@ static int fdt_resv_memory_update_node(void *fdt, unsigned long addr,
 	if (subnode < 0)
 		return subnode;
 
-	if (no_map) {
-		/*
-		 * Tell operating system not to create a virtual
-		 * mapping of the region as part of its standard
-		 * mapping of system memory.
-		 */
-		err = fdt_setprop_empty(fdt, subnode, "no-map");
-		if (err < 0)
-			return err;
-	}
+	/*
+	 * Tell operating system not to create a virtual
+	 * mapping of the region as part of its standard
+	 * mapping of system memory.
+	 */
+	err = fdt_setprop_empty(fdt, subnode, "no-map");
+	if (err < 0)
+		return err;
 
 	/* encode the <reg> property value */
 	val = reg;
@@ -199,9 +284,10 @@ int fdt_reserved_memory_fixup(void *fdt)
 {
 	struct sbi_domain_memregion *reg;
 	struct sbi_domain *dom = sbi_domain_thishart_ptr();
-	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	unsigned long filtered_base[PMP_COUNT] = { 0 };
+	unsigned char filtered_order[PMP_COUNT] = { 0 };
 	unsigned long addr, size;
-	int err, parent, i;
+	int err, parent, i, j;
 	int na = fdt_address_cells(fdt, 0);
 	int ns = fdt_size_cells(fdt, 0);
 
@@ -259,42 +345,41 @@ int fdt_reserved_memory_fixup(void *fdt)
 		/* Ignore MMIO or READABLE or WRITABLE or EXECUTABLE regions */
 		if (reg->flags & SBI_DOMAIN_MEMREGION_MMIO)
 			continue;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_READABLE)
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_READABLE)
 			continue;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_WRITEABLE)
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE)
 			continue;
-		if (reg->flags & SBI_DOMAIN_MEMREGION_EXECUTABLE)
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
 			continue;
 
+		if (i >= PMP_COUNT) {
+			sbi_printf("%s: Too many memory regions to fixup.\n",
+				   __func__);
+			return SBI_ENOSPC;
+		}
+
+		bool overlap = false;
 		addr = reg->base;
-		size = 1UL << reg->order;
-		fdt_resv_memory_update_node(fdt, addr, size, i, parent,
-			(sbi_hart_pmp_count(scratch)) ? false : true);
-		i++;
+		for (j = 0; j < i; j++) {
+			if (addr == filtered_base[j]
+			    && filtered_order[j] < reg->order) {
+				overlap = true;
+				filtered_order[j] = reg->order;
+				break;
+			}
+		}
+
+		if (!overlap) {
+			filtered_base[i] = reg->base;
+			filtered_order[i] = reg->order;
+			i++;
+		}
 	}
 
-	return 0;
-}
-
-int fdt_reserved_memory_nomap_fixup(void *fdt)
-{
-	int parent, subnode;
-	int err;
-
-	/* Locate the reserved memory node */
-	parent = fdt_path_offset(fdt, "/reserved-memory");
-	if (parent < 0)
-		return parent;
-
-	fdt_for_each_subnode(subnode, fdt, parent) {
-		/*
-		 * Tell operating system not to create a virtual
-		 * mapping of the region as part of its standard
-		 * mapping of system memory.
-		 */
-		err = fdt_setprop_empty(fdt, subnode, "no-map");
-		if (err < 0)
-			return err;
+	for (j = 0; j < i; j++) {
+		addr = filtered_base[j];
+		size = 1UL << filtered_order[j];
+		fdt_resv_memory_update_node(fdt, addr, size, j, parent);
 	}
 
 	return 0;
@@ -309,5 +394,8 @@ void fdt_fixups(void *fdt)
 	fdt_plic_fixup(fdt);
 
 	fdt_reserved_memory_fixup(fdt);
+
+#ifndef CONFIG_FDT_FIXUPS_PRESERVE_PMU_NODE
 	fdt_pmu_fixup(fdt);
+#endif
 }

lib/utils/fdt/fdt_helper.c

@@ -12,6 +12,7 @@
 #include <sbi/sbi_hartmask.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_scratch.h>
+#include <sbi/sbi_hart.h>
 #include <sbi_utils/fdt/fdt_helper.h>
 #include <sbi_utils/irqchip/aplic.h>
 #include <sbi_utils/irqchip/imsic.h>
@@ -215,6 +216,32 @@ int fdt_get_node_addr_size(void *fdt, int node, int index,
 	return 0;
 }
 
+int fdt_get_node_addr_size_by_name(void *fdt, int node, const char *name,
+				   uint64_t *addr, uint64_t *size)
+{
+	int i, j, count;
+	const char *val;
+	const char *regname;
+
+	if (!fdt || node < 0 || !name)
+		return SBI_EINVAL;
+
+	val = fdt_getprop(fdt, node, "reg-names", &count);
+	if (!val)
+		return SBI_ENODEV;
+
+	for (i = 0, j = 0; i < count; i++, j++) {
+		regname = val + i;
+
+		if (strcmp(name, regname) == 0)
+			return fdt_get_node_addr_size(fdt, node, j, addr, size);
+
+		i += strlen(regname);
+	}
+
+	return SBI_ENODEV;
+}
+
 bool fdt_node_is_enabled(void *fdt, int nodeoff)
 {
 	int len;
@@ -313,6 +340,149 @@ int fdt_parse_timebase_frequency(void *fdt, unsigned long *freq)
 	return 0;
 }
 
+#define RISCV_ISA_EXT_NAME_LEN_MAX	32
+
+static unsigned long fdt_isa_bitmap_offset;
+
+static int fdt_parse_isa_one_hart(const char *isa, unsigned long *extensions)
+{
+	size_t i, j, isa_len;
+	char mstr[RISCV_ISA_EXT_NAME_LEN_MAX];
+
+	i = 0;
+	isa_len = strlen(isa);
+
+	if (isa[i] == 'r' || isa[i] == 'R')
+		i++;
+	else
+		return SBI_EINVAL;
+
+	if (isa[i] == 'v' || isa[i] == 'V')
+		i++;
+	else
+		return SBI_EINVAL;
+
+	if (isa[i] == '3' || isa[i+1] == '2')
+		i += 2;
+	else if (isa[i] == '6' || isa[i+1] == '4')
+		i += 2;
+	else
+		return SBI_EINVAL;
+
+	/* Skip base ISA extensions */
+	for (; i < isa_len; i++) {
+		if (isa[i] == '_')
+			break;
+	}
+
+	while (i < isa_len) {
+		if (isa[i] != '_') {
+			i++;
+			continue;
+		}
+
+		/* Skip the '_' character */
+		i++;
+
+		/* Extract the multi-letter extension name */
+		j = 0;
+		while ((i < isa_len) && (isa[i] != '_') &&
+		       (j < (sizeof(mstr) - 1)))
+			mstr[j++] = isa[i++];
+		mstr[j] = '\0';
+
+		/* Skip empty multi-letter extension name */
+		if (!j)
+			continue;
+
+#define set_multi_letter_ext(name, bit)				\
+			if (!strcmp(mstr, name)) {		\
+				__set_bit(bit, extensions);	\
+				continue;			\
+			}
+
+		for (j = 0; j < SBI_HART_EXT_MAX; j++) {
+			set_multi_letter_ext(sbi_hart_ext[j].name,
+					     sbi_hart_ext[j].id);
+		}
+#undef set_multi_letter_ext
+	}
+
+	return 0;
+}
+
+static int fdt_parse_isa_all_harts(void *fdt)
+{
+	u32 hartid;
+	const fdt32_t *val;
+	unsigned long *hart_exts;
+	struct sbi_scratch *scratch;
+	int err, cpu_offset, cpus_offset, len;
+
+	if (!fdt || !fdt_isa_bitmap_offset)
+		return SBI_EINVAL;
+
+	cpus_offset = fdt_path_offset(fdt, "/cpus");
+	if (cpus_offset < 0)
+		return cpus_offset;
+
+	fdt_for_each_subnode(cpu_offset, fdt, cpus_offset) {
+		err = fdt_parse_hart_id(fdt, cpu_offset, &hartid);
+		if (err)
+			continue;
+
+		if (!fdt_node_is_enabled(fdt, cpu_offset))
+			continue;
+
+		val = fdt_getprop(fdt, cpu_offset, "riscv,isa", &len);
+		if (!val || len <= 0)
+			return SBI_ENOENT;
+
+		scratch = sbi_hartid_to_scratch(hartid);
+		if (!scratch)
+			return SBI_ENOENT;
+
+		hart_exts = sbi_scratch_offset_ptr(scratch,
+						   fdt_isa_bitmap_offset);
+
+		err = fdt_parse_isa_one_hart((const char *)val, hart_exts);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+int fdt_parse_isa_extensions(void *fdt, unsigned int hartid,
+			unsigned long *extensions)
+{
+	int rc, i;
+	unsigned long *hart_exts;
+	struct sbi_scratch *scratch;
+
+	if (!fdt_isa_bitmap_offset) {
+		fdt_isa_bitmap_offset = sbi_scratch_alloc_offset(
+					sizeof(*hart_exts) *
+					BITS_TO_LONGS(SBI_HART_EXT_MAX));
+		if (!fdt_isa_bitmap_offset)
+			return SBI_ENOMEM;
+
+		rc = fdt_parse_isa_all_harts(fdt);
+		if (rc)
+			return rc;
+	}
+
+	scratch = sbi_hartid_to_scratch(hartid);
+	if (!scratch)
+		return SBI_ENOENT;
+
+	hart_exts = sbi_scratch_offset_ptr(scratch, fdt_isa_bitmap_offset);
+
+	for (i = 0; i < BITS_TO_LONGS(SBI_HART_EXT_MAX); i++)
+		extensions[i] |= hart_exts[i];
+	return 0;
+}
+
 static int fdt_parse_uart_node_common(void *fdt, int nodeoffset,
 				      struct platform_uart_data *uart,
 				      unsigned long default_freq,
@@ -689,7 +859,7 @@ int fdt_parse_imsic_node(void *fdt, int nodeoff, struct imsic_data *imsic)
 			break;
 		regs->addr = reg_addr;
 		regs->size = reg_size;
-	};
+	}
 	if (!imsic->regs[0].size)
 		return SBI_EINVAL;
 
@@ -710,6 +880,7 @@ int fdt_parse_plic_node(void *fdt, int nodeoffset, struct plic_data *plic)
 	if (rc < 0 || !reg_addr || !reg_size)
 		return SBI_ENODEV;
 	plic->addr = reg_addr;
+	plic->size = reg_size;
 
 	val = fdt_getprop(fdt, nodeoffset, "riscv,ndev", &len);
 	if (len > 0)
@@ -732,21 +903,40 @@ int fdt_parse_plic(void *fdt, struct plic_data *plic, const char *compat)
 	return fdt_parse_plic_node(fdt, nodeoffset, plic);
 }
 
-int fdt_parse_aclint_node(void *fdt, int nodeoffset, bool for_timer,
-			  unsigned long *out_addr1, unsigned long *out_size1,
-			  unsigned long *out_addr2, unsigned long *out_size2,
-			  u32 *out_first_hartid, u32 *out_hart_count)
+static int fdt_get_aclint_addr_size_by_name(void *fdt, int nodeoffset,
+					    unsigned long *out_addr1,
+					    unsigned long *out_size1,
+					    unsigned long *out_addr2,
+					    unsigned long *out_size2)
 {
-	const fdt32_t *val;
+	int rc;
 	uint64_t reg_addr, reg_size;
-	int i, rc, count, cpu_offset, cpu_intc_offset;
-	u32 phandle, hwirq, hartid, first_hartid, last_hartid, hart_count;
-	u32 match_hwirq = (for_timer) ? IRQ_M_TIMER : IRQ_M_SOFT;
 
-	if (nodeoffset < 0 || !fdt ||
-	    !out_addr1 || !out_size1 ||
-	    !out_first_hartid || !out_hart_count)
-		return SBI_EINVAL;
+	rc = fdt_get_node_addr_size_by_name(fdt, nodeoffset, "mtime",
+					    &reg_addr, &reg_size);
+	if (rc < 0 || !reg_size)
+		reg_addr = reg_size = 0;
+	*out_addr1 = reg_addr;
+	*out_size1 = reg_size;
+
+	rc = fdt_get_node_addr_size_by_name(fdt, nodeoffset, "mtimecmp",
+					    &reg_addr, &reg_size);
+	if (rc < 0 || !reg_size)
+		return SBI_ENODEV;
+	*out_addr2 = reg_addr;
+	*out_size2 = reg_size;
+
+	return 0;
+}
+
+static int fdt_get_aclint_addr_size(void *fdt, int nodeoffset,
+				    unsigned long *out_addr1,
+				    unsigned long *out_size1,
+				    unsigned long *out_addr2,
+				    unsigned long *out_size2)
+{
+	int rc;
+	uint64_t reg_addr, reg_size;
 
 	rc = fdt_get_node_addr_size(fdt, nodeoffset, 0,
 				    &reg_addr, &reg_size);
@@ -764,6 +954,37 @@ int fdt_parse_aclint_node(void *fdt, int nodeoffset, bool for_timer,
 	if (out_size2)
 		*out_size2 = reg_size;
 
+	return 0;
+}
+
+int fdt_parse_aclint_node(void *fdt, int nodeoffset,
+			  bool for_timer, bool allow_regname,
+			  unsigned long *out_addr1, unsigned long *out_size1,
+			  unsigned long *out_addr2, unsigned long *out_size2,
+			  u32 *out_first_hartid, u32 *out_hart_count)
+{
+	const fdt32_t *val;
+	int i, rc, count, cpu_offset, cpu_intc_offset;
+	u32 phandle, hwirq, hartid, first_hartid, last_hartid, hart_count;
+	u32 match_hwirq = (for_timer) ? IRQ_M_TIMER : IRQ_M_SOFT;
+
+	if (nodeoffset < 0 || !fdt ||
+	    !out_addr1 || !out_size1 ||
+	    !out_first_hartid || !out_hart_count)
+		return SBI_EINVAL;
+
+	if (for_timer && allow_regname && out_addr2 && out_size2 &&
+	    fdt_getprop(fdt, nodeoffset, "reg-names", NULL))
+		rc = fdt_get_aclint_addr_size_by_name(fdt, nodeoffset,
+						      out_addr1, out_size1,
+						      out_addr2, out_size2);
+	else
+		rc = fdt_get_aclint_addr_size(fdt, nodeoffset,
+					      out_addr1, out_size1,
+					      out_addr2, out_size2);
+	if (rc)
+		return rc;
+
 	*out_first_hartid = 0;
 	*out_hart_count = 0;
 

lib/utils/fdt/fdt_pmu.c

@@ -12,24 +12,21 @@
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_pmu.h>
+#include <sbi/sbi_scratch.h>
 #include <sbi_utils/fdt/fdt_helper.h>
+#include <sbi_utils/fdt/fdt_pmu.h>
 
 #define FDT_PMU_HW_EVENT_MAX (SBI_PMU_HW_EVENT_MAX * 2)
 
-struct fdt_pmu_hw_event_select {
-	uint32_t eidx;
-	uint64_t select;
-};
-
-static struct fdt_pmu_hw_event_select fdt_pmu_evt_select[FDT_PMU_HW_EVENT_MAX] = {0};
-static uint32_t hw_event_count;
+struct fdt_pmu_hw_event_select_map fdt_pmu_evt_select[FDT_PMU_HW_EVENT_MAX] = {0};
+uint32_t hw_event_count;
 
 uint64_t fdt_pmu_get_select_value(uint32_t event_idx)
 {
 	int i;
-	struct fdt_pmu_hw_event_select *event;
+	struct fdt_pmu_hw_event_select_map *event;
 
-	for (i = 0; i < SBI_PMU_HW_EVENT_MAX; i++) {
+	for (i = 0; i < hw_event_count; i++) {
 		event = &fdt_pmu_evt_select[i];
 		if (event->eidx == event_idx)
 			return event->select;
@@ -64,7 +61,7 @@ int fdt_pmu_setup(void *fdt)
 	int i, pmu_offset, len, result;
 	const u32 *event_val;
 	const u32 *event_ctr_map;
-	struct fdt_pmu_hw_event_select *event;
+	struct fdt_pmu_hw_event_select_map *event;
 	uint64_t raw_selector, select_mask;
 	u32 event_idx_start, event_idx_end, ctr_map;
 
@@ -73,7 +70,7 @@ int fdt_pmu_setup(void *fdt)
 
 	pmu_offset = fdt_node_offset_by_compatible(fdt, -1, "riscv,pmu");
 	if (pmu_offset < 0)
-		return SBI_EFAIL;
+		return SBI_ENOENT;
 
 	event_ctr_map = fdt_getprop(fdt, pmu_offset,
 				    "riscv,event-to-mhpmcounters", &len);